feat(roundtable): 第二/三步模型自动容错(出问题自动换下一个模型)

模型出问题时按 config.yaml models[] 顺序自动切到下一个模型继续,并告知用户
「『X』模型异常 → 已切换『Y』继续」。前台实时 + 后台挂起两条路径都覆盖。

- 中间件兜底/熔断 AIMessage 打 LLM_ERROR_MARKER 标记,is_llm_error_message 可识别
- roundtable_model_fallback: fallback_model_chain 候选链 + reason_text + 总开关
  env ROUNDTABLE_MODEL_FALLBACK
- 触发: 异常(start_run失败/idle卡死) / 兜底错误文案 / 交付为空(仅seat/leader)
- 后台 InProcessRoundtableGateway._run_turn_with_fallback 包住每轮 run
- 前台 _special_run/_leader_run 收尾检测失败→model_switch 帧+换模型重流
- 结构化抽取 flow_extract round1 astream 同款换模型循环
- Step2 单席位异常隔离: 单席位失败不拖垮整批/整链/整个作业
- 时间线插「系统提示」气泡 + toast + 诊断 model_switch 事件
- Tests: test_roundtable_model_fallback.py

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-20 21:37:45 +08:00
1 parent ae2706dad3
commit d383b0e9dd
18 files changed
+1033 -174

No files matched your search

@@ -24,6 +24,63 @@ from deerflow.config.app_config import AppConfig
logger = logging.getLogger(__name__)
# 兜底/熔断的 AIMessage 会带上 ``additional_kwargs[LLM_ERROR_MARKER] = reason``,
# 让下游(圆桌会商的单轮执行器等)能**可靠识别**「这条 AI 回复其实是模型调用失败的
# 兜底文案」而不是真实交付——据此触发自动换模型。``reason`` ∈ {quota, auth, busy,
# transient, upstream, generic, circuit}。注意:model_dump() 会保留 additional_kwargs,
# 故经 checkpointer 序列化后仍在;万一被剥离,``is_llm_error_message`` 还会回退按已知
# 兜底文案精确匹配。
LLM_ERROR_MARKER = "deerflow_llm_error"
# 各 reason 对应的用户可见兜底文案(单一数据源:构造与识别都用它)。
_USER_MESSAGES: dict[str, str] = {
"quota": "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。",
"auth": "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。",
"busy": "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。",
"transient": "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。",
"upstream": "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。",
"generic": "大模型本次调用未能成功完成,请稍后再试或更换模型。",
}
_CIRCUIT_MESSAGE = "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
# 全部兜底文案集合,供 ``is_llm_error_message`` 在 additional_kwargs 被剥离时按正文回退识别。
_ALL_ERROR_MESSAGES: frozenset[str] = frozenset({*_USER_MESSAGES.values(), _CIRCUIT_MESSAGE})
def is_llm_error_message(message: Any) -> str | None:
"""判断一条 AI 消息是否为「模型调用失败的兜底文案」,是则返回 reason,否则 None。
入参可为 LangChain ``BaseMessage``,或 checkpointer 序列化后的 dict(两种都支持)。
优先看 ``additional_kwargs[LLM_ERROR_MARKER]``(最可靠);万一被序列化剥离,再回退
按正文是否精确等于已知兜底文案识别(容忍前后空白)。
"""
if message is None:
return None
# additional_kwargs(dict 或对象属性两种形态)。
extra: Any = None
if isinstance(message, dict):
extra = message.get("additional_kwargs")
else:
extra = getattr(message, "additional_kwargs", None)
if isinstance(extra, dict):
reason = extra.get(LLM_ERROR_MARKER)
if reason:
return str(reason)
# 正文回退匹配。
if isinstance(message, dict):
content = message.get("content")
else:
content = getattr(message, "content", None)
text = ""
if isinstance(content, str):
text = content
elif isinstance(content, list):
text = "\n".join(p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text")
text = text.strip()
if text and text in _ALL_ERROR_MESSAGES:
return "generic"
return None
_RETRIABLE_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504}
_BUSY_PATTERNS = (
"server busy",
@@ -201,17 +258,9 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
def _build_circuit_breaker_message(self) -> str:
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
return _CIRCUIT_MESSAGE
def _build_user_message(self, reason: str) -> str:
if reason == "quota":
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
if reason == "auth":
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
if reason in {"busy", "transient"}:
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
if reason == "upstream":
return "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。"
# NOTE: We intentionally do NOT embed the raw exception detail into the
# user-facing message. Raw `Error code: 400 - {...}` strings — especially
# Python repr-style dict literals — get persisted as AI messages and then
@@ -219,7 +268,11 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
# braces can corrupt the upstream JSON payload and cause deterministic
# JSONDecodeError loops. Keep the user message clean; full detail is
# already in the backend log via `logger.warning(..., exc_info=exc)`.
return "大模型本次调用未能成功完成,请稍后再试或更换模型。"
return _USER_MESSAGES.get(reason, _USER_MESSAGES["generic"])
def _error_message(self, content: str, reason: str) -> AIMessage:
"""构造带 ``LLM_ERROR_MARKER`` 标记的兜底 AIMessage(供下游识别+换模型)。"""
return AIMessage(content=content, additional_kwargs={LLM_ERROR_MARKER: reason})
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
try:
@@ -246,7 +299,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
if self._check_circuit():
return AIMessage(content=self._build_circuit_breaker_message())
return self._error_message(self._build_circuit_breaker_message(), "circuit")
attempt = 1
while True:
@@ -283,7 +336,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(reason))
return self._error_message(self._build_user_message(reason), reason)
@override
async def awrap_model_call(
@@ -292,7 +345,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
if self._check_circuit():
return AIMessage(content=self._build_circuit_breaker_message())
return self._error_message(self._build_circuit_breaker_message(), "circuit")
attempt = 1
while True:
@@ -329,7 +382,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(reason))
return self._error_message(self._build_user_message(reason), reason)
def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool:
@@ -202,6 +202,26 @@ _SEAT_NO_ASK = (
)
async def _emit_switches(
progress: JobProgressWriter,
switches: list,
*,
agent_name: str,
cycle: int = 0,
agent_id: str | None = None,
) -> None:
"""把单轮执行器带回的模型容错记录逐条插进研讨时间线(系统提示气泡)。"""
for sw in switches or []:
await progress.model_switch(
agent_name=agent_name,
failed_model=getattr(sw, "failed_model", ""),
next_model=getattr(sw, "next_model", ""),
reason_label=getattr(sw, "reason_label", "调用失败"),
cycle=cycle,
agent_id=agent_id,
)
def _deliveries_block(deliveries: dict[str, str], name_by_id: dict[str, str]) -> str:
parts: list[str] = []
for aid, text in deliveries.items():
@@ -292,6 +312,7 @@ async def _run_gather_phase(
logger.warning("gather phase seat %s failed", seat.agent_id, exc_info=True)
await progress.seat_done(seat.agent_id, 0)
return
await _emit_switches(progress, seat_turn.switches, agent_name=name_by_id.get(seat.agent_id, seat.agent_id), agent_id=seat.agent_id)
text = (seat_turn.final_text or "").strip()
if text:
gathered[seat.agent_id] = text
@@ -335,6 +356,7 @@ async def _run_recommend(
message=message,
model=model,
)
await _emit_switches(progress, turn.switches, agent_name="总控", cycle=cycle)
if turn.final_text:
last_leader = turn.final_text
await progress.add_leader_dialogue(
@@ -357,18 +379,32 @@ async def _run_recommend(
batch: dict[str, str] = {}
async def _run_one(agent_id: str, task: str, cyc: int = cycle) -> None:
name = name_by_id.get(agent_id, agent_id)
await progress.seat_active(agent_id)
seat_msg = _seat_message(intent_text, prior_snapshot, name_by_id, task, gathered=gathered)
seat = await gateway.run_seat(thread_ids=thread_ids, agent_id=agent_id, task=seat_msg, model=model)
try:
seat = await gateway.run_seat(thread_ids=thread_ids, agent_id=agent_id, task=seat_msg, model=model)
except Exception: # noqa: BLE001 — 单席位彻底失败不拖垮整批;置空交付 + 记诊断
logger.warning("recommend seat %s failed", agent_id, exc_info=True)
await progress.diag(
stage="step2_seat", level="error", event="seat_failed",
message=f"席位「{name}」本轮执行异常(已跳过,不影响其它席位)",
agent_id=agent_id, agent_name=name, cycle=cyc,
)
batch[agent_id] = ""
await progress.add_seat_dialogue(agent_id, name, "", cyc, thread_id=thread_ids.get(agent_id))
await progress.seat_done(agent_id, cyc)
return
await _emit_switches(progress, seat.switches, agent_name=name, cycle=cyc, agent_id=agent_id)
batch[agent_id] = seat.final_text
if not (seat.final_text or "").strip():
await progress.diag(
stage="step2_seat", level="warning", event="seat_empty_delivery",
message=f"席位「{name_by_id.get(agent_id, agent_id)}」本轮交付为空(可能模型未产出或被截断)",
agent_id=agent_id, agent_name=name_by_id.get(agent_id, agent_id), cycle=cyc,
message=f"席位「{name}」本轮交付为空(可能模型未产出或被截断)",
agent_id=agent_id, agent_name=name, cycle=cyc,
)
await progress.add_seat_dialogue(
agent_id, name_by_id.get(agent_id, agent_id), seat.final_text, cyc,
agent_id, name, seat.final_text, cyc,
tool_calls=seat.tool_calls, thread_id=thread_ids.get(agent_id),
)
await progress.seat_done(agent_id, cyc)
@@ -426,6 +462,7 @@ async def _run_chain(
message=leader_task,
model=model,
)
await _emit_switches(progress, turn.switches, agent_name="总控", cycle=i + 1)
if turn.final_text:
last_leader = turn.final_text
await progress.add_leader_dialogue(
@@ -440,7 +477,22 @@ async def _run_chain(
# chain 严格串行(seat B 看 seat A 交付),但有 gathered 时各席位已带共享资料,不必各自从零检索 → 提速。
await progress.seat_active(seat.agent_id)
seat_msg = _seat_message(intent_text, deliveries, name_by_id, task, gathered=gathered)
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=seat_msg, model=model)
try:
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=seat_msg, model=model)
except Exception: # noqa: BLE001 — 单席位失败不中断整链,置空交付后继续后续席位
logger.warning("chain seat %s failed", seat.agent_id, exc_info=True)
await progress.diag(
stage="step2_seat", level="error", event="seat_failed",
message=f"席位「{seat.name}」本轮执行异常(已跳过,链条继续)",
agent_id=seat.agent_id, agent_name=seat.name, cycle=i + 1,
)
deliveries[seat.agent_id] = ""
await progress.add_seat_dialogue(seat.agent_id, seat.name, "", i + 1, thread_id=thread_ids.get(seat.agent_id))
await progress.seat_done(seat.agent_id, i + 1)
if total > 0:
await progress.set_consensus_pct(round((i + 1) / total * 90))
continue
await _emit_switches(progress, seat_turn.switches, agent_name=seat.name, cycle=i + 1, agent_id=seat.agent_id)
deliveries[seat.agent_id] = seat_turn.final_text
if not (seat_turn.final_text or "").strip():
await progress.diag(
@@ -468,6 +520,7 @@ async def _run_chain(
message=closeout_msg,
model=model,
)
await _emit_switches(progress, closeout.switches, agent_name="总控", cycle=total + 1)
if closeout.final_text:
last_leader = closeout.final_text
await progress.add_leader_dialogue(
@@ -532,6 +585,7 @@ async def _run_dag(
message=leader_task,
model=model,
)
await _emit_switches(progress, turn.switches, agent_name="总控", cycle=i + 1)
if turn.final_text:
last_leader = turn.final_text
await progress.add_leader_dialogue(
@@ -545,22 +599,35 @@ async def _run_dag(
for seat in stage_seats:
await progress.seat_active(seat.agent_id)
async def _run_one(seat: SeatRef, _gathered: dict[str, str] | None = gathered) -> None:
async def _run_one(seat: SeatRef, _gathered: dict[str, str] | None = gathered, _cyc: int = i + 1) -> None:
task = pick_dag_seat_task(turn.dispatched, seat.agent_id, seat.name, goal=stage.goal)
seat_msg = _seat_message(intent_text, deliveries, name_by_id, task, gathered=_gathered)
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=seat_msg, model=model)
try:
seat_turn = await gateway.run_seat(thread_ids=thread_ids, agent_id=seat.agent_id, task=seat_msg, model=model)
except Exception: # noqa: BLE001 — 单席位失败不拖垮整个 stage 并行批
logger.warning("dag seat %s failed", seat.agent_id, exc_info=True)
await progress.diag(
stage="step2_seat", level="error", event="seat_failed",
message=f"席位「{seat.name}」本轮执行异常(已跳过,不影响同批其它席位)",
agent_id=seat.agent_id, agent_name=seat.name, cycle=_cyc,
)
deliveries[seat.agent_id] = ""
await progress.add_seat_dialogue(seat.agent_id, seat.name, "", _cyc, thread_id=thread_ids.get(seat.agent_id))
await progress.seat_done(seat.agent_id, _cyc)
return
await _emit_switches(progress, seat_turn.switches, agent_name=seat.name, cycle=_cyc, agent_id=seat.agent_id)
deliveries[seat.agent_id] = seat_turn.final_text
if not (seat_turn.final_text or "").strip():
await progress.diag(
stage="step2_seat", level="warning", event="seat_empty_delivery",
message=f"席位「{seat.name}」本轮交付为空(可能模型未产出或被截断)",
agent_id=seat.agent_id, agent_name=seat.name, cycle=i + 1,
agent_id=seat.agent_id, agent_name=seat.name, cycle=_cyc,
)
await progress.add_seat_dialogue(
seat.agent_id, seat.name, seat_turn.final_text, i + 1,
seat.agent_id, seat.name, seat_turn.final_text, _cyc,
tool_calls=seat_turn.tool_calls, thread_id=thread_ids.get(seat.agent_id),
)
await progress.seat_done(seat.agent_id, i + 1)
await progress.seat_done(seat.agent_id, _cyc)
await asyncio.gather(*(_run_one(seat) for seat in stage_seats))
@@ -578,6 +645,7 @@ async def _run_dag(
message=closeout_msg,
model=model,
)
await _emit_switches(progress, closeout.switches, agent_name="总控", cycle=total + 1)
if closeout.final_text:
last_leader = closeout.final_text
await progress.add_leader_dialogue(
@@ -627,6 +695,7 @@ async def _run_step3(
summary: SummaryResult = await gateway.run_summary(
summary_thread_id=summary_thread_id, prompt=summary_prompt, model=model
)
await _emit_switches(progress, getattr(summary, "switches", None), agent_name="方案总结报告")
if (summary.md or "").strip() or (summary.closing or "").strip():
step3["summaryReport"] = {
"md": summary.md or "",
@@ -710,6 +779,7 @@ async def _run_report(
report_thread_id = await gateway.create_report_thread()
prompt = build_report_prompt(intent_text, agents, deliveries, consensus_text)
report: ReportResult = await gateway.run_report(report_thread_id=report_thread_id, prompt=prompt, model=model)
await _emit_switches(progress, getattr(report, "switches", None), agent_name="结果绘制")
return {
"html": report.html,
"generatedAt": report.generated_at or datetime.now(UTC).isoformat(),
@@ -270,3 +270,27 @@ class JobProgressWriter:
self._add("seat", agent_id, name, content or "", cycle, tool_calls=tool_calls, thread_id=thread_id)
await self._persist()
await self._flush_dialogue()
async def model_switch(
self,
*,
agent_name: str,
failed_model: str,
next_model: str,
reason_label: str,
cycle: int = 0,
agent_id: str | None = None,
) -> None:
"""模型自动容错通知:当前模型出问题、已自动切到下一个模型时调用。
往研讨时间线插一条 ``role="system"`` 系统提示气泡(前端 ``toStep2Dialogue`` 据此渲染
为「系统提示」)。对应的 ``model_switch`` **诊断**由网关
(``InProcessRoundtableGateway._run_turn_with_fallback``)在换模型当下落库(含完整
detail),此处只负责时间线展示,避免重复落诊断。``reason_label`` 由调用方(app 层)
预先翻译好(harness 不依赖 app 的文案表)。"""
fm = (failed_model or "").strip() or "当前模型"
nm = (next_model or "").strip() or "下一个模型"
content = f"⚠️ 模型容错:「{agent_name}」所用模型「{fm}」{reason_label},已自动切换到「{nm}」继续。"
self._add("system", agent_id, "系统提示", content, cycle)
await self._persist()
await self._flush_dialogue()
@@ -43,6 +43,20 @@ class SeatRef:
name: str
@dataclass
class ModelSwitch:
"""一次「模型自动容错」记录:当前模型出问题、已切到下一个模型继续。
由网关(单轮执行器)在换模型时产出,挂在各 Turn/Result 上带回引擎;引擎据此调
``JobProgressWriter.model_switch(...)`` 往研讨时间线插系统提示并落诊断。``reason_label``
已由 app 层翻译成简短中文(harness 不依赖 app 文案表)。
"""
failed_model: str
next_model: str
reason_label: str
@dataclass
class LeaderTurn:
"""总控一轮执行的归一化结果(由 gateway.run_leader 产出)。"""
@@ -55,6 +69,8 @@ class LeaderTurn:
# 本轮总控调用的工具(按出现顺序,每项 {name, args, id})——通常是 agent_orchestration
# 派活 / ask_clarification。用于在草稿 run 里重建「工具步骤卡」,与前端手动跑一致。
tool_calls: list[dict] = field(default_factory=list)
# 本轮发生的模型自动容错记录(按发生顺序);引擎据此向用户提示「已切换模型」。
switches: list[ModelSwitch] = field(default_factory=list)
@property
def is_clarification(self) -> bool:
@@ -73,6 +89,8 @@ class SeatTurn:
# 本轮席位调用的工具(按出现顺序,每项 {name, args, id})——用于在草稿 run 里
# 重建「工具步骤卡」+ write_file stub(点开产物),与前端手动跑一致。
tool_calls: list[dict] = field(default_factory=list)
# 本轮发生的模型自动容错记录(按发生顺序)。
switches: list[ModelSwitch] = field(default_factory=list)
@dataclass
@@ -83,6 +101,7 @@ class ReportResult:
summary: str | None = None
model: str | None = None
generated_at: str | None = None
switches: list[ModelSwitch] = field(default_factory=list)
@dataclass
@@ -99,6 +118,7 @@ class SummaryResult:
model: str | None = None
thread_id: str | None = None
generated_at: str | None = None
switches: list[ModelSwitch] = field(default_factory=list)
# ── 网关协议 ────────────────────────────────────────────────────────────────