feat(roundtable): 第二/三步模型自动容错(出问题自动换下一个模型)

模型出问题时按 config.yaml models[] 顺序自动切到下一个模型继续,并告知用户
「『X』模型异常 → 已切换『Y』继续」。前台实时 + 后台挂起两条路径都覆盖。

- 中间件兜底/熔断 AIMessage 打 LLM_ERROR_MARKER 标记,is_llm_error_message 可识别
- roundtable_model_fallback: fallback_model_chain 候选链 + reason_text + 总开关
  env ROUNDTABLE_MODEL_FALLBACK
- 触发: 异常(start_run失败/idle卡死) / 兜底错误文案 / 交付为空(仅seat/leader)
- 后台 InProcessRoundtableGateway._run_turn_with_fallback 包住每轮 run
- 前台 _special_run/_leader_run 收尾检测失败→model_switch 帧+换模型重流
- 结构化抽取 flow_extract round1 astream 同款换模型循环
- Step2 单席位异常隔离: 单席位失败不拖垮整批/整链/整个作业
- 时间线插「系统提示」气泡 + toast + 诊断 model_switch 事件
- Tests: test_roundtable_model_fallback.py

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-20 21:37:45 +08:00
1 parent ae2706dad3
commit d383b0e9dd
18 files changed
+1033 -174

No files matched your search

@@ -18,12 +18,18 @@ import asyncio
import logging
import os
import uuid
from collections.abc import Callable
from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
from langgraph.checkpoint.base import empty_checkpoint
from app.gateway.roundtable_model_fallback import (
fallback_model_chain,
is_llm_error_message,
reason_text,
)
from app.gateway.roundtable_run_policy import roundtable_run_policy
from app.gateway.roundtable_seat_skills import append_seat_skill_directive
from app.gateway.routers._roundtable_seed import (
@@ -36,6 +42,7 @@ from app.gateway.routers._roundtable_seed import (
from deerflow.agents.roundtable_orchestrator.report_json import extract_report_json
from deerflow.agents.roundtable_orchestrator.types import (
LeaderTurn,
ModelSwitch,
ReportResult,
SeatRef,
SeatTurn,
@@ -521,6 +528,31 @@ _SEAT_MODE_PARAMS: dict[str, tuple[bool | None, str | None, bool]] = {
}
def _exception_reason(exc: BaseException) -> str:
"""把单轮 run 抛出的异常归类成模型容错原因码(用于换模型提示)。
idle 守护判定的卡死(``_await_run_with_idle_guard`` 抛 ``... stalled ...``)→ ``timeout``;
其余(start_run 失败 / DB 抖动 / 其它)→ ``exception``。
"""
text = str(exc).lower()
if "stall" in text or "idle" in text or "timeout" in text or "timed out" in text:
return "timeout"
return "exception"
def _seat_empty_reason(messages: list[dict[str, Any]]) -> str | None:
"""席位/取数轮校验:最后一条有正文的 AI 消息为空 → ``empty``(触发换模型)。"""
return "empty" if not _latest_ai_text(messages).strip() else None
def _leader_failed_reason(messages: list[dict[str, Any]]) -> str | None:
"""总控轮校验:既无正文、也无任何工具调用(派活/澄清)→ ``empty``(视为模型没干活)。"""
ai = _latest_ai_message(messages)
has_text = bool(_latest_ai_text(messages).strip())
has_tools = bool(_normalize_tool_calls((ai or {}).get("tool_calls")))
return None if (has_text or has_tools) else "empty"
# ── 网关实现 ─────────────────────────────────────────────────────────────────
@@ -585,25 +617,116 @@ class InProcessRoundtableGateway:
self._seat_intro = _build_seat_intro(self._agents)
return thread_ids, COORDINATOR_AGENT_ID
async def _run_turn_with_fallback(
self,
*,
thread_id: str,
agent_name: str,
message: str,
requested_model: str | None,
excluded_tools: list[str],
skill_stop_names: list[str],
thinking_enabled: bool,
reasoning_effort: str,
subagent_enabled: bool = False,
stage: str,
cycle: int | None = None,
validate: Callable[[list[dict[str, Any]]], str | None] | None = None,
) -> tuple[list[dict[str, Any]], list[ModelSwitch], str | None]:
"""跑一轮 agent,并在「模型出问题」时自动切到下一个模型重试。
触发换模型的三种情形(与用户选择一致):
① ``_run_agent_turn`` 抛异常(start_run 失败 / idle 卡死超时);
② 返回了中间件兜底错误文案(``is_llm_error_message`` 命中);
③ 调用方 ``validate`` 判定无效(如席位交付为空 / 总控既无正文又无派活)。
候选模型链按 ``config.yaml models[]`` 顺序(``fallback_model_chain``),首个为
``requested_model``(空则作业级 ``self._model``,再空则配置默认)。返回
``(最终 messages, 发生的换模型记录, 实际所用模型)``。全部模型都失败时返回最后一次的
messages(让既有的「空交付 / 产物缺失」逻辑兜底,绝不抛异常拖垮整个作业)。
"""
base = requested_model if (requested_model not in (None, "")) else self._model
chain = fallback_model_chain(base) or [base]
switches: list[ModelSwitch] = []
messages: list[dict[str, Any]] = []
prev_model: str | None = None
prev_reason: str | None = None
used_model: str | None = chain[0]
for idx, candidate in enumerate(chain):
if idx > 0:
label = reason_text(prev_reason)
switches.append(ModelSwitch(failed_model=prev_model or "", next_model=candidate or "", reason_label=label))
logger.warning(
"roundtable %s '%s' switching model %s -> %s (%s)",
stage, agent_name, prev_model, candidate, prev_reason,
)
await self._diag.record(
stage=stage, level="warning", event="model_switch",
message=f"「{agent_name}」模型「{prev_model}」{label},已自动切换到「{candidate}」重试",
detail={"failed_model": prev_model, "next_model": candidate, "reason": prev_reason, "attempt": idx + 1},
agent_name=agent_name, cycle=cycle,
)
used_model = candidate
try:
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
thread_id=thread_id,
agent_name=agent_name,
message=message,
model=candidate,
excluded_tools=excluded_tools,
skill_stop_names=skill_stop_names,
thinking_enabled=thinking_enabled,
reasoning_effort=reasoning_effort,
subagent_enabled=subagent_enabled,
diag=self._diag,
stage=stage,
cycle=cycle,
)
except Exception as exc: # noqa: BLE001 — 模型故障:归类后尝试下一个模型
prev_model, prev_reason, messages = candidate, _exception_reason(exc), []
if idx + 1 < len(chain):
continue
await self._diag.record(
stage=stage, level="error", event="model_fallback_exhausted",
message=f"「{agent_name}」所有备选模型均失败(最后 {candidate}:{prev_reason})",
detail={"reason": prev_reason, "tried": chain}, agent_name=agent_name, cycle=cycle,
)
return messages, switches, used_model
# 跑通了:再看是否「兜底错误文案」或 validate 判定无效。
reason = is_llm_error_message(_latest_ai_message(messages))
if reason is None and validate is not None:
reason = validate(messages)
if reason is None:
return messages, switches, used_model
prev_model, prev_reason = candidate, reason
if idx + 1 >= len(chain):
await self._diag.record(
stage=stage, level="error", event="model_fallback_exhausted",
message=f"「{agent_name}」所有备选模型均未产出有效结果(最后 {candidate}:{reason})",
detail={"reason": reason, "tried": chain}, agent_name=agent_name, cycle=cycle,
)
return messages, switches, used_model
return messages, switches, used_model
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
msg = message
if self._first_leader and self._seat_intro:
msg = self._seat_intro + "\n\n" + message
self._first_leader = False
policy = roundtable_run_policy("leader")
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
messages, switches, _used = await self._run_turn_with_fallback(
thread_id=thread_ids[coordinator_name],
agent_name=coordinator_name,
message=msg,
model=model or self._model,
requested_model=model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
diag=self._diag,
stage="step2_leader",
validate=_leader_failed_reason,
)
ai = _latest_ai_message(messages)
# tool_calls 取最后一条 AI 消息(派活就在这条);正文取最后一条有文字的 AI 消息。
@@ -617,7 +740,7 @@ class InProcessRoundtableGateway:
if clar is not None:
cargs = clar.get("args") or {}
question = str(cargs.get("question") or "").strip() or "总控需要补充信息,请补充后继续。"
return LeaderTurn(final_text=final_text, clarification=question, tool_calls=turn_tool_calls)
return LeaderTurn(final_text=final_text, clarification=question, tool_calls=turn_tool_calls, switches=switches)
# 派活解析(复用 multi_agent 纯函数)
from app.gateway.routers.multi_agent import _extract_dispatch
@@ -630,7 +753,7 @@ class InProcessRoundtableGateway:
dispatched.append((name, task))
except Exception: # noqa: BLE001
logger.warning("dispatch parse failed for tool_call", exc_info=True)
return LeaderTurn(final_text=final_text, dispatched=dispatched, tool_calls=turn_tool_calls)
return LeaderTurn(final_text=final_text, dispatched=dispatched, tool_calls=turn_tool_calls, switches=switches)
async def run_seat(self, *, thread_ids, agent_id, task, model) -> SeatTurn:
policy = roundtable_run_policy("seat")
@@ -654,13 +777,11 @@ class InProcessRoundtableGateway:
override = _SEAT_MODE_PARAMS.get(self._seat_modes.get(agent_id, ""))
if override is not None:
seat_thinking, seat_reasoning, seat_subagent = override
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
messages, switches, _used = await self._run_turn_with_fallback(
thread_id=thread_ids[agent_id],
agent_name=agent_id,
message=seat_task,
model=model or self._model,
requested_model=model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
thinking_enabled=(
@@ -670,10 +791,14 @@ class InProcessRoundtableGateway:
policy.reasoning_effort if seat_reasoning is None else seat_reasoning
),
subagent_enabled=seat_subagent,
diag=self._diag,
stage="step2_seat",
validate=_seat_empty_reason,
)
return SeatTurn(
final_text=_latest_ai_text(messages),
tool_calls=_tool_calls_since_last_human(messages),
switches=switches,
)
return SeatTurn(final_text=_latest_ai_text(messages), tool_calls=_tool_calls_since_last_human(messages))
async def create_report_thread(self) -> str:
ensure_roundtable_functional_agents()
@@ -686,21 +811,22 @@ class InProcessRoundtableGateway:
summary = ""
html: str | None = None
message = prompt
switches: list[ModelSwitch] = []
current_model = model # 跨产物自愈轮记住实际所用模型:一旦换到可用模型,后续续写轮就从它起。
for attempt in range(1 + _ARTIFACT_RETRY_LIMIT):
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
messages, sw, used = await self._run_turn_with_fallback(
thread_id=report_thread_id,
agent_name=REPORT_AGENT_ID,
message=message,
model=model or self._model,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
diag=self._diag,
stage="step3_report",
)
switches.extend(sw)
current_model = used or current_model
summary = _latest_ai_text(messages) or summary
html = _read_report_html(self._app, report_thread_id, self._user_id)
if _html_complete(html):
@@ -727,7 +853,7 @@ class InProcessRoundtableGateway:
)
# 截断的 HTML 也比纯文字强(浏览器容忍未闭合标签),仅在完全没有文件时才回退正文。
final_html = html if (html and html.strip()) else summary
return ReportResult(html=final_html, summary=summary, model=model or self._model)
return ReportResult(html=final_html, summary=summary, model=current_model or self._model, switches=switches)
async def create_summary_thread(self) -> str:
ensure_roundtable_functional_agents()
@@ -743,21 +869,22 @@ class InProcessRoundtableGateway:
closing = ""
md: str | None = None
message = prompt
switches: list[ModelSwitch] = []
current_model = model
for attempt in range(1 + _ARTIFACT_RETRY_LIMIT):
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
messages, sw, used = await self._run_turn_with_fallback(
thread_id=summary_thread_id,
agent_name=SUMMARY_AGENT_ID,
message=message,
model=model or self._model,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
diag=self._diag,
stage="step3_summary",
)
switches.extend(sw)
current_model = used or current_model
closing = _latest_ai_text(messages) or closing
md = _read_summary_md(self._app, summary_thread_id, self._user_id)
if md and md.strip():
@@ -783,7 +910,7 @@ class InProcessRoundtableGateway:
agent_name=SUMMARY_AGENT_ID,
)
final_md = md if (md and md.strip()) else closing
return SummaryResult(md=final_md, closing=closing, model=model or self._model, thread_id=summary_thread_id)
return SummaryResult(md=final_md, closing=closing, model=current_model or self._model, thread_id=summary_thread_id, switches=switches)
async def run_dashboard_data(self, *, prompt, model) -> str | None:
"""大屏多页的结构化数据:进程内驱动**专职**「大屏多页数据智能体」
@@ -799,21 +926,20 @@ class InProcessRoundtableGateway:
thread_id = await _create_thread(self._app)
policy = roundtable_run_policy("dashboard")
message = prompt
current_model = model
for attempt in range(1 + _ARTIFACT_RETRY_LIMIT):
messages = await _run_agent_turn(
self._app,
user_id=self._user_id,
messages, _sw, used = await self._run_turn_with_fallback(
thread_id=thread_id,
agent_name=DASHBOARD_AGENT_ID,
message=message,
model=model or self._model,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
diag=self._diag,
stage="step3_dashboard",
)
current_model = used or current_model
body = _extract_report_json(_latest_ai_text(messages))
if body:
return body