feat(roundtable): 第二/三步模型自动容错(出问题自动换下一个模型)

模型出问题时按 config.yaml models[] 顺序自动切到下一个模型继续,并告知用户
「『X』模型异常 → 已切换『Y』继续」。前台实时 + 后台挂起两条路径都覆盖。

- 中间件兜底/熔断 AIMessage 打 LLM_ERROR_MARKER 标记,is_llm_error_message 可识别
- roundtable_model_fallback: fallback_model_chain 候选链 + reason_text + 总开关
  env ROUNDTABLE_MODEL_FALLBACK
- 触发: 异常(start_run失败/idle卡死) / 兜底错误文案 / 交付为空(仅seat/leader)
- 后台 InProcessRoundtableGateway._run_turn_with_fallback 包住每轮 run
- 前台 _special_run/_leader_run 收尾检测失败→model_switch 帧+换模型重流
- 结构化抽取 flow_extract round1 astream 同款换模型循环
- Step2 单席位异常隔离: 单席位失败不拖垮整批/整链/整个作业
- 时间线插「系统提示」气泡 + toast + 诊断 model_switch 事件
- Tests: test_roundtable_model_fallback.py

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-20 21:37:45 +08:00
1 parent ae2706dad3
commit d383b0e9dd
18 files changed
+1033 -174

No files matched your search

@@ -24,6 +24,63 @@ from deerflow.config.app_config import AppConfig
logger = logging.getLogger(__name__)
# 兜底/熔断的 AIMessage 会带上 ``additional_kwargs[LLM_ERROR_MARKER] = reason``,
# 让下游(圆桌会商的单轮执行器等)能**可靠识别**「这条 AI 回复其实是模型调用失败的
# 兜底文案」而不是真实交付——据此触发自动换模型。``reason`` ∈ {quota, auth, busy,
# transient, upstream, generic, circuit}。注意:model_dump() 会保留 additional_kwargs,
# 故经 checkpointer 序列化后仍在;万一被剥离,``is_llm_error_message`` 还会回退按已知
# 兜底文案精确匹配。
LLM_ERROR_MARKER = "deerflow_llm_error"
# 各 reason 对应的用户可见兜底文案(单一数据源:构造与识别都用它)。
_USER_MESSAGES: dict[str, str] = {
"quota": "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。",
"auth": "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。",
"busy": "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。",
"transient": "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。",
"upstream": "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。",
"generic": "大模型本次调用未能成功完成,请稍后再试或更换模型。",
}
_CIRCUIT_MESSAGE = "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
# 全部兜底文案集合,供 ``is_llm_error_message`` 在 additional_kwargs 被剥离时按正文回退识别。
_ALL_ERROR_MESSAGES: frozenset[str] = frozenset({*_USER_MESSAGES.values(), _CIRCUIT_MESSAGE})
def is_llm_error_message(message: Any) -> str | None:
"""判断一条 AI 消息是否为「模型调用失败的兜底文案」,是则返回 reason,否则 None。
入参可为 LangChain ``BaseMessage``,或 checkpointer 序列化后的 dict(两种都支持)。
优先看 ``additional_kwargs[LLM_ERROR_MARKER]``(最可靠);万一被序列化剥离,再回退
按正文是否精确等于已知兜底文案识别(容忍前后空白)。
"""
if message is None:
return None
# additional_kwargs(dict 或对象属性两种形态)。
extra: Any = None
if isinstance(message, dict):
extra = message.get("additional_kwargs")
else:
extra = getattr(message, "additional_kwargs", None)
if isinstance(extra, dict):
reason = extra.get(LLM_ERROR_MARKER)
if reason:
return str(reason)
# 正文回退匹配。
if isinstance(message, dict):
content = message.get("content")
else:
content = getattr(message, "content", None)
text = ""
if isinstance(content, str):
text = content
elif isinstance(content, list):
text = "\n".join(p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text")
text = text.strip()
if text and text in _ALL_ERROR_MESSAGES:
return "generic"
return None
_RETRIABLE_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504}
_BUSY_PATTERNS = (
"server busy",
@@ -201,17 +258,9 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
return f"大模型因{reason_text}调用失败,正在第 {attempt}/{self.retry_max_attempts} 次重试,{seconds} 秒后继续…"
def _build_circuit_breaker_message(self) -> str:
return "大模型因连续多次调用失败已被熔断保护,请稍候再试。"
return _CIRCUIT_MESSAGE
def _build_user_message(self, reason: str) -> str:
if reason == "quota":
return "大模型因账户额度不足、账单异常或使用受限而拒绝请求,请检查模型账户后再试。"
if reason == "auth":
return "大模型因鉴权失败或无访问权限而拒绝请求,请检查模型的 API Key 与权限配置后再试。"
if reason in {"busy", "transient"}:
return "大模型因服务繁忙或网络异常,多次重试后仍未恢复,请稍候再继续。"
if reason == "upstream":
return "大模型当前响应异常(上游模型或网关临时故障),多次重试后仍未恢复,请稍后再试或更换模型。"
# NOTE: We intentionally do NOT embed the raw exception detail into the
# user-facing message. Raw `Error code: 400 - {...}` strings — especially
# Python repr-style dict literals — get persisted as AI messages and then
@@ -219,7 +268,11 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
# braces can corrupt the upstream JSON payload and cause deterministic
# JSONDecodeError loops. Keep the user message clean; full detail is
# already in the backend log via `logger.warning(..., exc_info=exc)`.
return "大模型本次调用未能成功完成,请稍后再试或更换模型。"
return _USER_MESSAGES.get(reason, _USER_MESSAGES["generic"])
def _error_message(self, content: str, reason: str) -> AIMessage:
"""构造带 ``LLM_ERROR_MARKER`` 标记的兜底 AIMessage(供下游识别+换模型)。"""
return AIMessage(content=content, additional_kwargs={LLM_ERROR_MARKER: reason})
def _emit_retry_event(self, attempt: int, wait_ms: int, reason: str) -> None:
try:
@@ -246,7 +299,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelCallResult:
if self._check_circuit():
return AIMessage(content=self._build_circuit_breaker_message())
return self._error_message(self._build_circuit_breaker_message(), "circuit")
attempt = 1
while True:
@@ -283,7 +336,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(reason))
return self._error_message(self._build_user_message(reason), reason)
@override
async def awrap_model_call(
@@ -292,7 +345,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelCallResult:
if self._check_circuit():
return AIMessage(content=self._build_circuit_breaker_message())
return self._error_message(self._build_circuit_breaker_message(), "circuit")
attempt = 1
while True:
@@ -329,7 +382,7 @@ class LLMErrorHandlingMiddleware(AgentMiddleware[AgentState]):
)
if retriable:
self._record_failure()
return AIMessage(content=self._build_user_message(reason))
return self._error_message(self._build_user_message(reason), reason)
def _matches_any(detail: str, patterns: tuple[str, ...]) -> bool: