feat(roundtable): 第二/三步模型自动容错(出问题自动换下一个模型)

模型出问题时按 config.yaml models[] 顺序自动切到下一个模型继续,并告知用户
「『X』模型异常 → 已切换『Y』继续」。前台实时 + 后台挂起两条路径都覆盖。

- 中间件兜底/熔断 AIMessage 打 LLM_ERROR_MARKER 标记,is_llm_error_message 可识别
- roundtable_model_fallback: fallback_model_chain 候选链 + reason_text + 总开关
  env ROUNDTABLE_MODEL_FALLBACK
- 触发: 异常(start_run失败/idle卡死) / 兜底错误文案 / 交付为空(仅seat/leader)
- 后台 InProcessRoundtableGateway._run_turn_with_fallback 包住每轮 run
- 前台 _special_run/_leader_run 收尾检测失败→model_switch 帧+换模型重流
- 结构化抽取 flow_extract round1 astream 同款换模型循环
- Step2 单席位异常隔离: 单席位失败不拖垮整批/整链/整个作业
- 时间线插「系统提示」气泡 + toast + 诊断 model_switch 事件
- Tests: test_roundtable_model_fallback.py

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
董宏凯andClaude Opus 4.8 committed 2026-06-20 21:37:45 +08:00
1 parent ae2706dad3
commit d383b0e9dd
18 files changed
+1033 -174

No files matched your search

@@ -31,12 +31,12 @@ from fastapi.responses import StreamingResponse
from langchain_core.messages import HumanMessage, SystemMessage
from pydantic import BaseModel, Field
from deerflow.agents.roundtable_orchestrator.extraction_spec import build_structure_directive
from app.gateway.roundtable_model_fallback import fallback_model_chain, reason_text
from app.gateway.routers._roundtable_seed import (
STRUCTURE_AGENT_ID,
ensure_roundtable_functional_agents,
)
from deerflow.agents.roundtable_orchestrator.extraction_spec import build_structure_directive
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config, load_agent_soul
from deerflow.models.factory import create_chat_model
@@ -596,28 +596,60 @@ async def extract_flow_stream(request: Request, payload: FlowExtractRequest = Bo
return
# ① 抽取 / 编辑(正文 + 思考(reasoning) 拼成 `<think>…</think>正文` 增量逐帧吐,前端实时显示)
# 模型自动容错:结构化抽取非常重要——某模型 astream 抛错 / 产出空时,按 config.yaml
# models[] 顺序切到下一个模型重抽,并发 model_switch 帧让前端把抽取面板已累计的半截内容重置。
round1_messages = _edit_messages(payload) if is_edit else _extract_messages(payload)
model = create_chat_model(name=resolved_model, thinking_enabled=True)
chain = fallback_model_chain(resolved_model) or [resolved_model]
extracted = ""
think_open = False
async for chunk in model.astream(round1_messages):
c, r = _chunk_parts(chunk)
out = ""
if r:
if not think_open:
out += "<think>"
think_open = True
out += r
if c:
used_model = chain[0]
prev_model: str | None = None
prev_reason: str | None = None
for idx, candidate in enumerate(chain):
if idx > 0:
yield _frame(
{
"event": "model_switch",
"agent_name": "结构化抽取",
"failed_model": prev_model or "",
"next_model": candidate or "",
"reason": reason_text(prev_reason),
}
)
used_model = candidate
extracted = ""
think_open = False
try:
model = create_chat_model(name=candidate, thinking_enabled=True)
async for chunk in model.astream(round1_messages):
c, r = _chunk_parts(chunk)
out = ""
if r:
if not think_open:
out += "<think>"
think_open = True
out += r
if c:
if think_open:
out += "</think>"
think_open = False
out += c
extracted += c
if out:
yield _frame({"event": "extract_delta", "content": out})
if think_open:
out += "</think>"
think_open = False
out += c
extracted += c
if out:
yield _frame({"event": "extract_delta", "content": out})
if think_open:
yield _frame({"event": "extract_delta", "content": "</think>"})
yield _frame({"event": "extract_delta", "content": "</think>"})
except Exception as exc: # noqa: BLE001 — 该模型抽取失败:换下一个模型重抽
prev_model, prev_reason = candidate, "exception"
logger.warning("flow-extract round1 model %s failed: %s", candidate, exc)
if idx + 1 < len(chain):
continue
raise
if extracted.strip():
break
# 产出为空:换下一个模型(已是最后一个则继续走后续解析兜底)。
prev_model, prev_reason = candidate, "empty"
if idx + 1 >= len(chain):
break
yield _frame({"event": "extract_done"})
# ② 校验 / 修复(同样流式 + 思考实时显示)
@@ -625,7 +657,7 @@ async def extract_flow_stream(request: Request, payload: FlowExtractRequest = Bo
validated = ""
graph: dict[str, Any] | None = None
try:
vmodel = create_chat_model(name=resolved_model, thinking_enabled=True)
vmodel = create_chat_model(name=used_model, thinking_enabled=True)
think_open = False
async for chunk in vmodel.astream(_validate_messages(payload, extracted)):
c, r = _chunk_parts(chunk)
@@ -63,6 +63,7 @@ from fastapi.responses import StreamingResponse
from langgraph.checkpoint.base import empty_checkpoint
from app.gateway.roundtable_diag import record_foreground_diag
from app.gateway.roundtable_model_fallback import fallback_model_chain, is_llm_error_message, reason_text
from app.gateway.roundtable_run_policy import merge_skill_stop_names, roundtable_run_policy
from deerflow.agents.roundtable_orchestrator.extraction_spec import build_summary_directive
@@ -1547,21 +1548,31 @@ async def _leader_run(
if files:
_read_tools = {"read_file", "ls", "grep", "view_image"}
leader_excluded = [t for t in leader_excluded if t not in _read_tools]
body = _run_payload(
agent_name=agent_name,
model_name=model_name,
thread_id=thread_id,
# 注入了席位清单的输入(leader_input);seat_roster 为空时等同 new_message。
new_message=leader_input,
excluded_tools=leader_excluded,
skill_stop_names=effective_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
# 人工干预/恢复会重发 leader run 到同一总控 thread;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
# 用户随干预上传的参考文件 → 总控可用 read_file 等工具读取后再派活。
files=files,
)
# 模型自动容错:按 config.yaml models[] 顺序的候选链。当前模型出问题(兜底错误文案 /
# 既无正文又无派活)时切到下一个模型重跑(见循环内检测)。``_mk_body`` 用候选模型重建 body。
model_chain = fallback_model_chain(model_name) or [model_name]
model_idx = 0
current_model = model_chain[0]
def _mk_body(m: str) -> Any:
return _run_payload(
agent_name=agent_name,
model_name=m,
thread_id=thread_id,
# 注入了席位清单的输入(leader_input);seat_roster 为空时等同 new_message。
new_message=leader_input,
excluded_tools=leader_excluded,
skill_stop_names=effective_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
# 人工干预/恢复会重发 leader run 到同一总控 thread;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
# 用户随干预上传的参考文件 → 总控可用 read_file 等工具读取后再派活。
files=files,
)
body = _mk_body(current_model)
t_payload_built = time.perf_counter()
logger.info(
"[leader-timing] %s payload_built=%.3fs (broadcast running in background)",
@@ -1645,6 +1656,28 @@ async def _leader_run(
leader_text = _flatten_content(dispatch_msg.get("content", ""))
tool_calls = dispatch_msg.get("tool_calls") or []
# 模型自动容错:本轮总控产出是「中间件兜底错误文案」或「既无正文又无派活」(模型没干活)
# → 若还有备选模型,切到下一个重跑(不消耗派活纠正预算)。先发 model_switch 帧让前端把
# 这条失败气泡重置,再用新模型重新流式。最后一个模型仍失败则照常往下走(错误文案当共识展示)。
fail_reason = is_llm_error_message(dispatch_msg)
if fail_reason is None and not (leader_text or "").strip() and not tool_calls:
fail_reason = "empty"
if fail_reason and model_idx + 1 < len(model_chain):
model_idx += 1
next_model = model_chain[model_idx]
logger.warning("[leader-fallback] %s 模型 %s 失败(%s),切换到 %s", agent_name, current_model, fail_reason, next_model)
yield {
"status": "model_switch",
"role": "leader",
"agent_name": agent_name,
"failed_model": current_model,
"next_model": next_model,
"reason": reason_text(fail_reason),
}
current_model = next_model
body = _mk_body(current_model)
continue
# 澄清分支:与 intent.py 相同字段;status 为字符串 "clarification"(非数组),
# 前端据此暂停 runOrchestration 循环,展示 Step 2 澄清条。
clarification_call = next(
@@ -1665,12 +1698,13 @@ async def _leader_run(
"allow_custom": bool(cargs.get("allow_custom", True)),
"allow_multiple": resolve_allow_multiple(cargs),
"agent_name": agent_name,
"model_used": current_model,
}
return
if not tool_calls:
# 空 status 数组 = 总控不再派活,前端视为共识(hasConsensus)。
yield {"status": [], "content": leader_text, "agent_name": agent_name}
yield {"status": [], "content": leader_text, "agent_name": agent_name, "model_used": current_model}
return
# 派活前按真实 roster 校验:合法保留、非法丢弃(只丢非法,不殃及同轮合法席位)。
@@ -1706,7 +1740,7 @@ async def _leader_run(
# 关键路径优先:**先 yield 派活帧**,让前端立刻启动席位 special run;派活广播(告知
# 其它席位"总控给 X 派了什么活")改成 detached 后台任务,不再 await 阻塞 —— 否则广播
# 读写各席位巨大 thread 状态会卡几十秒,白白拖慢席位启动(见 _spawn_detached_broadcast)。
yield {"status": dispatched, "content": leader_text, "agent_name": agent_name}
yield {"status": dispatched, "content": leader_text, "agent_name": agent_name, "model_used": current_model}
# DAG 派活轮(suppress_pre_broadcast)同样**不**把「总控给 X 派活」广播给兄弟席位 ——
# 否则弱模型从历史里看到派活语句又会模仿派活(席位任务由前端 pickDagSeatTask 干净下发,
# 跨席位承上靠各 special run 的"完成交付"后广播,不需要这条派活广播)。recommend/chain 照旧。
@@ -1805,95 +1839,128 @@ async def _special_run(
directive = build_summary_directive(business_code)
if directive:
seat_message = f"{new_message}\n\n---\n\n{directive}"
body = _run_payload(
agent_name=agent_name,
model_name=model_name,
thread_id=thread_id,
new_message=seat_message,
excluded_tools=seat_excluded,
skill_stop_names=merge_skill_stop_names(policy.skill_stop_names, skill_stop_names),
# 席位执行模式覆盖 policy 的 thinking/reasoning(None → 回退 policy 默认);
# subagent 仅 ultra 档置真。工具裁剪(excluded_tools)仍完全由 policy 决定。
thinking_enabled=(policy.thinking_enabled if thinking_enabled is None else thinking_enabled),
reasoning_effort=(policy.reasoning_effort if reasoning_effort is None else reasoning_effort),
subagent_enabled=subagent_enabled,
# ultra 席位才会派子智能体 → 才需要 custom 流回传子任务实时进度(task_running 等)。
stream_custom=subagent_enabled,
# 干预/重新派活会把同一席位 thread 再跑一次;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
# 前序席位完整交付 → run context,供 read_peer_delivery 工具按需取全文(§12.9)。
extra_context=(
{"roundtable_peer_deliveries": peer_deliveries} if peer_deliveries else None
),
# 用户随干预上传的参考文件 → 席位可用 read_file 等工具读取。
files=files,
)
# 模型自动容错:按 config.yaml models[] 顺序的候选链。某模型出问题(兜底错误文案;
# 真正的研讨席位还含「交付为空」)→ 发 model_switch 帧让前端重置该席位气泡,再用下一个
# 模型重跑。Step3 单例(report/summary/dashboard/ingest)的产物是落盘文件,正文可能很短,
# 故**不**把「正文为空」当失败(与后台 run_report/summary 的 validate=None 一致),只在真正
# 的模型错误时换模型。
model_chain = fallback_model_chain(model_name) or [model_name]
empty_is_failure = role in ("seat", "seat_parallel")
def _mk_body(m: str) -> Any:
return _run_payload(
agent_name=agent_name,
model_name=m,
thread_id=thread_id,
new_message=seat_message,
excluded_tools=seat_excluded,
skill_stop_names=merge_skill_stop_names(policy.skill_stop_names, skill_stop_names),
# 席位执行模式覆盖 policy 的 thinking/reasoning(None → 回退 policy 默认);
# subagent 仅 ultra 档置真。工具裁剪(excluded_tools)仍完全由 policy 决定。
thinking_enabled=(policy.thinking_enabled if thinking_enabled is None else thinking_enabled),
reasoning_effort=(policy.reasoning_effort if reasoning_effort is None else reasoning_effort),
subagent_enabled=subagent_enabled,
# ultra 席位才会派子智能体 → 才需要 custom 流回传子任务实时进度(task_running 等)。
stream_custom=subagent_enabled,
# 干预/重新派活会把同一席位 thread 再跑一次;抢占掉没结束的旧 run,避免 409。
multitask_strategy="rollback",
# 前序席位完整交付 → run context,供 read_peer_delivery 工具按需取全文(§12.9)。
extra_context=(
{"roundtable_peer_deliveries": peer_deliveries} if peer_deliveries else None
),
# 用户随干预上传的参考文件 → 席位可用 read_file 等工具读取。
files=files,
)
t_payload_built = time.perf_counter()
logger.info(
"[special-timing] %s payload_built=%.3fs",
agent_name,
t_payload_built - t0,
)
captured: list[str] = []
first_line_ts: float | None = None
line_count = 0
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
if line is not None:
if first_line_ts is None:
first_line_ts = time.perf_counter()
logger.info(
"[special-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
agent_name,
first_line_ts - t_payload_built,
first_line_ts - t0,
)
line_count += 1
yield line
elif done is not None:
captured = done
t_stream_done = time.perf_counter()
logger.info(
"[special-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
agent_name,
t_stream_done - (first_line_ts or t_payload_built),
line_count,
t_stream_done - t0,
)
messages = _parse_last_messages(captured)
if messages is None:
# 与 _leader_run 同理:席位 run 被抢占/断开导致流截断,不抛 502,静默收尾。
# WARNING 级:正常是 rollback 抢占;否则查上游 run 的服务端报错(如 DB 连接断)。
logger.warning(
"[special-timing] %s upstream truncated (no values frame) — ending quietly "
"(expected on rollback preemption; otherwise check the upstream run's server-side error)",
agent_name,
)
return
# Sub-agent's final deliverable is the latest AIMessage's text content.
last_msg: dict[str, Any] = {}
for msg in reversed(messages):
if isinstance(msg, dict) and (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
last_msg = msg
break
content = _flatten_content(last_msg.get("content", ""))
# 优先用 display_name(中文友好,例如"情报收集")提示 leader,id 放括号备查;
# display_name 缺失时退化为仅 agent_name —— 与旧行为一致。
speaker_label = f"{display_name}({agent_name})" if display_name else agent_name
_prefix = f"子智能体{speaker_label}完成{new_message}工作,交付内容为:"
# 广播给**所有**席位(含总控)的都只是**截断摘要** —— 总控 thread 不再囤积各席位全文,
# 每轮 leader 的上下文因此保持小、响应快。总控做最终综合需要全文时,由 _leader_run 在
# 「综合轮」按需现读各席位 thread 注入(见 _build_full_deliverables_block),不靠历史囤积。
brief_msg = _truncate_broadcast_content(_prefix, content)
# 先 yield 交付结果让前端立即收口本席位;后广播改 detached,不再 await 阻塞 ——
# 否则它读写各席位大状态会让 SSE 多挂十几秒,把下一轮 leader 的启动也一起拖慢。
yield {
"status": f"子智能体{speaker_label}完成{new_message}工作",
"content": content,
"agent_name": agent_name,
}
_spawn_detached_broadcast(base, headers, brief_msg, agent_threads, [agent_name])
for idx, candidate in enumerate(model_chain):
body = _mk_body(candidate)
captured: list[str] = []
first_line_ts: float | None = None
line_count = 0
async for line, done in _stream_upstream(client, base, headers, thread_id, body):
if line is not None:
if first_line_ts is None:
first_line_ts = time.perf_counter()
logger.info(
"[special-timing] %s first_upstream_line=%.3fs since_entry=%.3fs",
agent_name,
first_line_ts - t_payload_built,
first_line_ts - t0,
)
line_count += 1
yield line
elif done is not None:
captured = done
t_stream_done = time.perf_counter()
logger.info(
"[special-timing] %s upstream_stream_complete=%.3fs lines=%d since_entry=%.3fs",
agent_name,
t_stream_done - (first_line_ts or t_payload_built),
line_count,
t_stream_done - t0,
)
messages = _parse_last_messages(captured)
if messages is None:
# 与 _leader_run 同理:席位 run 被抢占/断开导致流截断,不抛 502,静默收尾。
# 这不是模型错误(多半 rollback 抢占),不触发换模型。
logger.warning(
"[special-timing] %s upstream truncated (no values frame) — ending quietly "
"(expected on rollback preemption; otherwise check the upstream run's server-side error)",
agent_name,
)
return
# Sub-agent's final deliverable is the latest AIMessage's text content.
last_msg: dict[str, Any] = {}
for msg in reversed(messages):
if isinstance(msg, dict) and (msg.get("type") or "").lower() in ("ai", "aimessage", "aimessagechunk"):
last_msg = msg
break
content = _flatten_content(last_msg.get("content", ""))
# 模型失败检测:兜底错误文案;研讨席位再加「交付为空」。还有备选模型则换。
fail_reason = is_llm_error_message(last_msg)
if fail_reason is None and empty_is_failure and not content.strip():
fail_reason = "empty"
if fail_reason and idx + 1 < len(model_chain):
next_model = model_chain[idx + 1]
logger.warning("[special-fallback] %s 模型 %s 失败(%s),切换到 %s", agent_name, candidate, fail_reason, next_model)
yield {
"status": "model_switch",
"role": role,
"agent_name": agent_name,
"failed_model": candidate,
"next_model": next_model,
"reason": reason_text(fail_reason),
}
continue
# 成功(或已用尽备选):发交付终态帧 + detached 后广播。
_prefix = f"子智能体{speaker_label}完成{new_message}工作,交付内容为:"
# 广播给**所有**席位(含总控)的都只是**截断摘要** —— 总控 thread 不再囤积各席位全文,
# 每轮 leader 的上下文因此保持小、响应快。总控做最终综合需要全文时,由 _leader_run 在
# 「综合轮」按需现读各席位 thread 注入(见 _build_full_deliverables_block),不靠历史囤积。
brief_msg = _truncate_broadcast_content(_prefix, content)
# 先 yield 交付结果让前端立即收口本席位;后广播改 detached,不再 await 阻塞 ——
# 否则它读写各席位大状态会让 SSE 多挂十几秒,把下一轮 leader 的启动也一起拖慢。
yield {
"status": f"子智能体{speaker_label}完成{new_message}工作",
"content": content,
"agent_name": agent_name,
"model_used": candidate,
}
_spawn_detached_broadcast(base, headers, brief_msg, agent_threads, [agent_name])
return
# ---------------------------------------------------------------------------