feat(roundtable): 会商诊断日志 + 后台挂起卡死兜底(idle 超时)
- 新增会商全流程诊断日志:新表 roundtable_diagnostics + 仓储 + admin API (/api/roundtable-diagnostics) + 管理员诊断页(会商页右上角设置按钮入口), 覆盖第一/二/三步及后台执行器/进程内网关各处报错与关键事件,支持按 范围/阶段/级别/事件/关键词筛选 + 分页。诊断写入全程 best-effort,绝不打断会商。 - 修复后台挂起「执行完第一个席位后永久卡死」:进程内网关每轮 agent run 改为 idle(无活动)超时(默认 20min)——只要还在持续产出 token/工具事件就一直等, 仅当连续无任何输出才判卡死并取消+报错+落 seat_turn_timeout 诊断;慢但活着的 生成绝不打断。可经 ROUNDTABLE_SEAT_IDLE_TIMEOUT_SECONDS 调整。 - 一并提交进行中的会商 seat-skill 注入 / 业务链条编辑器相关改动。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8ca16c74bc
commit
4a403c95c5
27 files changed
+1873
-61
No files matched your search
@@ -25,6 +25,7 @@ from typing import Any
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
|
||||
from app.gateway.roundtable_run_policy import roundtable_run_policy
|
||||
from app.gateway.roundtable_seat_skills import append_seat_skill_directive
|
||||
from app.gateway.routers._roundtable_seed import (
|
||||
COORDINATOR_AGENT_ID,
|
||||
DASHBOARD_AGENT_ID,
|
||||
@@ -41,10 +42,31 @@ from deerflow.agents.roundtable_orchestrator.types import (
|
||||
SummaryResult,
|
||||
)
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.persistence.roundtable_diagnostics import DiagnosticsRecorder, get_default_store
|
||||
from deerflow.runtime.serialization import serialize_channel_values
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 单轮 agent run(leader/seat/report/summary/dashboard)的**无活动(idle)超时**(秒)。
|
||||
#
|
||||
# 背景:后台编排是**串行 await 每一个 run**,且历史上**没有任何超时保护**——只要某一轮
|
||||
# ``agent.astream`` 卡死(内网模型端点连接被打满 / sqlite 锁在挂载卷上挂死等环境相关原因),
|
||||
# 整个后台作业就永久冻结(用户侧表现为「执行完第一个席位后一直不动」)。
|
||||
#
|
||||
# ⚠️ 关键:这里**不是总时长超时**,而是 **idle 超时**——只要这轮 run 还在持续产出
|
||||
# (流式 token / 工具调用 / 进度事件,全部经 StreamBridge 外发),就一直等下去,哪怕这轮
|
||||
# 真的要跑 20 分钟、半小时也绝不打断(慢但活着的生成不该被切)。只有当 run **连续
|
||||
# ``_SEAT_IDLE_TIMEOUT_S`` 秒完全没有任何输出**(真卡死)时才取消它、抛错 → 顶层把 job 标
|
||||
# error(而非永久卡死)+ 落一条 seat_turn_timeout 诊断。默认 1200s(20 分钟)——内网模型
|
||||
# 很不稳定时单次「首 token / 工具间隔」可能拖很久,给足容忍度,宁可多等也别误杀慢生成;
|
||||
# idle 只兜真正「连续 20 分钟啥都不发生」的死锁。
|
||||
_SEAT_IDLE_TIMEOUT_S = int(os.getenv("ROUNDTABLE_SEAT_IDLE_TIMEOUT_SECONDS", "1200") or "1200")
|
||||
# 可选的绝对总时长上限(秒):默认 0 = **关闭**(只靠 idle 超时,慢生成不受限)。
|
||||
# 仅当你确实想给单轮设一个天花板时再设它(如某些场景要硬兜底)。
|
||||
_SEAT_MAX_TOTAL_S = int(os.getenv("ROUNDTABLE_SEAT_MAX_TOTAL_SECONDS", "0") or "0")
|
||||
# idle 检测的心跳间隔:每隔这么久检查一次「距上次活动过了多久」。
|
||||
_IDLE_CHECK_INTERVAL_S = 15.0
|
||||
|
||||
_ROUNDTABLE_THREAD_METADATA: dict[str, Any] = {"thread_type": "roundtable", "system": True}
|
||||
_REPORT_FILE = "方案总览.html"
|
||||
_SUMMARY_FILE = "方案总结报告.md"
|
||||
@@ -183,6 +205,108 @@ def _tool_calls_since_last_human(messages: list[dict[str, Any]]) -> list[dict[st
|
||||
return out
|
||||
|
||||
|
||||
async def _await_run_with_idle_guard(
|
||||
app,
|
||||
record,
|
||||
*,
|
||||
idle_timeout: float,
|
||||
diag: DiagnosticsRecorder | None,
|
||||
stage: str,
|
||||
agent_name: str,
|
||||
cycle: int | None,
|
||||
max_total: float = 0.0,
|
||||
) -> None:
|
||||
"""等待一轮 run 完成,但用 **无活动(idle) 超时**而非总时长超时。
|
||||
|
||||
只要 run 仍在持续产出(StreamBridge 上有新事件:流式 token / 工具调用 / 进度),就一直
|
||||
等下去——慢但活着的生成绝不打断。只有当 run **连续 ``idle_timeout`` 秒完全没有任何事件**
|
||||
(真卡死)时才取消它并抛 ``RuntimeError``。``max_total`` > 0 时附加一个绝对总时长上限。
|
||||
|
||||
活动信号取自 StreamBridge(worker 每产一块都 ``bridge.publish``)。拿不到 bridge / run_id
|
||||
时退化为「总时长超时(若配置)或直接等待」。
|
||||
"""
|
||||
task = getattr(record, "task", None)
|
||||
if task is None:
|
||||
return
|
||||
run_id = getattr(record, "run_id", None)
|
||||
bridge = getattr(app.state, "stream_bridge", None)
|
||||
loop = asyncio.get_running_loop()
|
||||
start = loop.time()
|
||||
|
||||
# 没有 bridge 可观察活动 → 退化:有 max_total 用之,否则直接等待(保持旧的「不卡死则一直等」)。
|
||||
if bridge is None or run_id is None:
|
||||
if max_total and max_total > 0:
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=max_total)
|
||||
else:
|
||||
await task
|
||||
return
|
||||
|
||||
from deerflow.runtime.stream_bridge.base import END_SENTINEL, HEARTBEAT_SENTINEL
|
||||
|
||||
last_activity = loop.time()
|
||||
stuck = asyncio.Event()
|
||||
|
||||
async def _watch() -> None:
|
||||
nonlocal last_activity
|
||||
async for ev in bridge.subscribe(run_id, heartbeat_interval=_IDLE_CHECK_INTERVAL_S):
|
||||
if ev is END_SENTINEL:
|
||||
return # run 已收尾,正常退出(不算卡死)
|
||||
now = loop.time()
|
||||
if ev is HEARTBEAT_SENTINEL:
|
||||
if now - last_activity >= idle_timeout:
|
||||
stuck.set()
|
||||
return
|
||||
if max_total and max_total > 0 and now - start >= max_total:
|
||||
stuck.set()
|
||||
return
|
||||
else:
|
||||
last_activity = now # 任何真实事件 = 还活着
|
||||
|
||||
watch = asyncio.create_task(_watch())
|
||||
try:
|
||||
done, _pending = await asyncio.wait({task, watch}, return_when=asyncio.FIRST_COMPLETED)
|
||||
if task in done:
|
||||
return # run 正常结束(run_agent 内部已吞掉异常并落终态)
|
||||
# watch 先完成:要么判定卡死,要么因 END 正常退出。
|
||||
if not stuck.is_set():
|
||||
await task # END 已出现,run 即将收尾,等它
|
||||
return
|
||||
# —— 判定卡死:取消该 run,落诊断,抛错 ——
|
||||
idle_elapsed = loop.time() - last_activity
|
||||
total = loop.time() - start
|
||||
logger.error(
|
||||
"roundtable %s turn '%s' stalled: no output for %.0fs (total %.0fs) on run %s",
|
||||
stage, agent_name, idle_elapsed, total, run_id,
|
||||
)
|
||||
run_manager = getattr(app.state, "run_manager", None)
|
||||
if run_manager is not None:
|
||||
try:
|
||||
await run_manager.cancel(run_id, action="interrupt")
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("cancel stalled run %s failed", run_id, exc_info=True)
|
||||
if diag is not None:
|
||||
await diag.record(
|
||||
stage=stage, level="error", event="seat_turn_timeout",
|
||||
message=(
|
||||
f"「{agent_name}」连续 {idle_elapsed:.0f}s 无任何输出、判定为卡死并已取消"
|
||||
f"(idle 上限 {idle_timeout:.0f}s;本轮已运行 {total:.0f}s)。常见原因:内网模型端点连接"
|
||||
"被打满 / sqlite checkpoint 在挂载卷上锁挂死。注意:慢但持续产出的生成不会被打断。"
|
||||
),
|
||||
detail={"idle_seconds": round(idle_elapsed, 1), "total_seconds": round(total, 1), "idle_timeout": idle_timeout},
|
||||
agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"roundtable seat turn '{agent_name}' stalled (no output for {idle_timeout:.0f}s)"
|
||||
) from None
|
||||
finally:
|
||||
if not watch.done():
|
||||
watch.cancel()
|
||||
try:
|
||||
await watch
|
||||
except BaseException: # noqa: BLE001 — 取消 watch 的异常无关紧要
|
||||
pass
|
||||
|
||||
|
||||
async def _run_agent_turn(
|
||||
app,
|
||||
*,
|
||||
@@ -196,6 +320,9 @@ async def _run_agent_turn(
|
||||
thinking_enabled: bool,
|
||||
reasoning_effort: str,
|
||||
subagent_enabled: bool = False,
|
||||
diag: DiagnosticsRecorder | None = None,
|
||||
stage: str = "step2_seat",
|
||||
cycle: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""进程内驱动 agent_name 在 thread_id 上跑一轮(发 message),阻塞到完成,返回最终 messages。
|
||||
|
||||
@@ -239,9 +366,25 @@ async def _run_agent_turn(
|
||||
# 取该用户的后台额度后再开跑:start_run + await task 是最重的部分(建 agent、调模型、
|
||||
# 落 checkpoint)。额度满时本轮在此 await 排队,等同用户某个在跑轮次释放额度后再开。
|
||||
async with _user_background_semaphore(user_id):
|
||||
record = await start_run(body, thread_id, request) # type: ignore[arg-type]
|
||||
if getattr(record, "task", None) is not None:
|
||||
await record.task # 阻塞到这轮 agent 跑完,checkpoint 已落盘
|
||||
try:
|
||||
record = await start_run(body, thread_id, request) # type: ignore[arg-type]
|
||||
except Exception as exc: # noqa: BLE001 — start_run 报错(409 抢占 / DB 抖动等)也要落诊断
|
||||
if diag is not None:
|
||||
await diag.record(
|
||||
stage=stage, level="error", event="start_run_failed",
|
||||
message=f"启动 agent run 失败({agent_name}):{exc}",
|
||||
detail={"type": type(exc).__name__, "error": str(exc), "thread_id": thread_id},
|
||||
agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
raise
|
||||
# idle 超时守护:慢但持续产出的生成一直等;只有连续 _SEAT_IDLE_TIMEOUT_S 秒**毫无输出**
|
||||
# (真卡死)才取消并抛错。绝不因「这轮总耗时长」而打断(除非显式配了 _SEAT_MAX_TOTAL_S)。
|
||||
await _await_run_with_idle_guard(
|
||||
app, record,
|
||||
idle_timeout=_SEAT_IDLE_TIMEOUT_S,
|
||||
max_total=_SEAT_MAX_TOTAL_S,
|
||||
diag=diag, stage=stage, agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
return await _read_thread_messages(app, thread_id)
|
||||
|
||||
|
||||
@@ -382,11 +525,23 @@ class InProcessRoundtableGateway:
|
||||
seat_thinking_enabled: bool | None = None,
|
||||
seat_reasoning_effort: str | None = None,
|
||||
seat_subagent_enabled: bool = False,
|
||||
job_id: str | None = None,
|
||||
draft_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._user_id = user_id
|
||||
self._agents = list(agents)
|
||||
self._model = model
|
||||
# 诊断日志记录器(绑定本作业上下文)。用进程级默认 store(deps.py 启动时已登记)。
|
||||
self._diag = DiagnosticsRecorder(
|
||||
get_default_store(),
|
||||
scope="background",
|
||||
job_id=job_id,
|
||||
draft_id=draft_id,
|
||||
task_id=task_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
# 席位执行模式(沿用 Step2 选择):覆盖 seat policy 的 thinking/reasoning(None → policy 默认),
|
||||
# subagent 仅 ultra 档为真。只作用 special 席位;leader 派活不受影响。
|
||||
self._seat_thinking_enabled = seat_thinking_enabled
|
||||
@@ -426,6 +581,8 @@ class InProcessRoundtableGateway:
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
diag=self._diag,
|
||||
stage="step2_leader",
|
||||
)
|
||||
ai = _latest_ai_message(messages)
|
||||
# tool_calls 取最后一条 AI 消息(派活就在这条);正文取最后一条有文字的 AI 消息。
|
||||
@@ -456,13 +613,17 @@ class InProcessRoundtableGateway:
|
||||
|
||||
async def run_seat(self, *, thread_ids, agent_id, task, model) -> SeatTurn:
|
||||
policy = roundtable_run_policy("seat")
|
||||
# 弱模型补偿:把该席位配置的技能(名称 + SKILL.md 路径 + 必须先 read_file 再按其
|
||||
# 流程取信息的硬指令)显式拼进本轮任务,避免弱模型无视系统提示里的技能块(见
|
||||
# roundtable_seat_skills 模块说明)。无配置技能时原样返回 task。
|
||||
seat_task = append_seat_skill_directive(agent_id, task)
|
||||
# 席位模式覆盖 policy 的 thinking/reasoning(None → policy 默认),与前端 _special_run 一致。
|
||||
messages = await _run_agent_turn(
|
||||
self._app,
|
||||
user_id=self._user_id,
|
||||
thread_id=thread_ids[agent_id],
|
||||
agent_name=agent_id,
|
||||
message=task,
|
||||
message=seat_task,
|
||||
model=model or self._model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
@@ -477,6 +638,8 @@ class InProcessRoundtableGateway:
|
||||
else self._seat_reasoning_effort
|
||||
),
|
||||
subagent_enabled=self._seat_subagent_enabled,
|
||||
diag=self._diag,
|
||||
stage="step2_seat",
|
||||
)
|
||||
return SeatTurn(final_text=_latest_ai_text(messages), tool_calls=_tool_calls_since_last_human(messages))
|
||||
|
||||
@@ -503,19 +666,32 @@ class InProcessRoundtableGateway:
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
diag=self._diag,
|
||||
stage="step3_report",
|
||||
)
|
||||
summary = _latest_ai_text(messages) or summary
|
||||
html = _read_report_html(self._app, report_thread_id, self._user_id)
|
||||
if _html_complete(html):
|
||||
break
|
||||
if attempt >= _ARTIFACT_RETRY_LIMIT:
|
||||
await self._diag.record(
|
||||
stage="step3_report", level="error", event="report_artifact_incomplete",
|
||||
message=f"结果绘制重试 {_ARTIFACT_RETRY_LIMIT} 次后 HTML 仍缺失/截断,回退为正文展示",
|
||||
detail={"thread_id": report_thread_id, "has_partial_html": bool(html and html.strip())},
|
||||
agent_name=REPORT_AGENT_ID,
|
||||
)
|
||||
break
|
||||
kind = "truncated" if (html and html.strip()) else "missing"
|
||||
message = _REPORT_RETRY_TRUNCATED if (html and html.strip()) else _REPORT_RETRY_MISSING
|
||||
logger.warning(
|
||||
"report html %s after attempt %d for thread %s; retrying with corrective prompt",
|
||||
"truncated" if (html and html.strip()) else "missing",
|
||||
attempt + 1,
|
||||
report_thread_id,
|
||||
kind, attempt + 1, report_thread_id,
|
||||
)
|
||||
await self._diag.record(
|
||||
stage="step3_report", level="warning", event=f"report_{kind}",
|
||||
message=f"结果绘制第 {attempt + 1} 次产物{('被截断' if kind == 'truncated' else '缺失')},已追加纠偏提示重试",
|
||||
detail={"thread_id": report_thread_id, "attempt": attempt + 1},
|
||||
agent_name=REPORT_AGENT_ID,
|
||||
)
|
||||
# 截断的 HTML 也比纯文字强(浏览器容忍未闭合标签),仅在完全没有文件时才回退正文。
|
||||
final_html = html if (html and html.strip()) else summary
|
||||
@@ -547,12 +723,20 @@ class InProcessRoundtableGateway:
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
diag=self._diag,
|
||||
stage="step3_summary",
|
||||
)
|
||||
closing = _latest_ai_text(messages) or closing
|
||||
md = _read_summary_md(self._app, summary_thread_id, self._user_id)
|
||||
if md and md.strip():
|
||||
break
|
||||
if attempt >= _ARTIFACT_RETRY_LIMIT:
|
||||
await self._diag.record(
|
||||
stage="step3_summary", level="error", event="summary_artifact_missing",
|
||||
message=f"方案总结报告重试 {_ARTIFACT_RETRY_LIMIT} 次后 .md 仍缺失,回退为对话正文",
|
||||
detail={"thread_id": summary_thread_id},
|
||||
agent_name=SUMMARY_AGENT_ID,
|
||||
)
|
||||
break
|
||||
message = _SUMMARY_RETRY_MISSING
|
||||
logger.warning(
|
||||
@@ -560,6 +744,12 @@ class InProcessRoundtableGateway:
|
||||
attempt + 1,
|
||||
summary_thread_id,
|
||||
)
|
||||
await self._diag.record(
|
||||
stage="step3_summary", level="warning", event="summary_missing",
|
||||
message=f"方案总结报告第 {attempt + 1} 次未生成 .md,已追加纠偏提示重试",
|
||||
detail={"thread_id": summary_thread_id, "attempt": attempt + 1},
|
||||
agent_name=SUMMARY_AGENT_ID,
|
||||
)
|
||||
final_md = md if (md and md.strip()) else closing
|
||||
return SummaryResult(md=final_md, closing=closing, model=model or self._model, thread_id=summary_thread_id)
|
||||
|
||||
@@ -589,11 +779,19 @@ class InProcessRoundtableGateway:
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
diag=self._diag,
|
||||
stage="step3_dashboard",
|
||||
)
|
||||
body = _extract_report_json(_latest_ai_text(messages))
|
||||
if body:
|
||||
return body
|
||||
if attempt >= _ARTIFACT_RETRY_LIMIT:
|
||||
await self._diag.record(
|
||||
stage="step3_dashboard", level="error", event="dashboard_json_invalid",
|
||||
message=f"大屏数据重试 {_ARTIFACT_RETRY_LIMIT} 次后仍抽取不到合法 report-json,放弃大屏产物",
|
||||
detail={"thread_id": thread_id},
|
||||
agent_name=DASHBOARD_AGENT_ID,
|
||||
)
|
||||
break
|
||||
message = _DASHBOARD_RETRY
|
||||
logger.warning(
|
||||
@@ -601,4 +799,10 @@ class InProcessRoundtableGateway:
|
||||
attempt + 1,
|
||||
thread_id,
|
||||
)
|
||||
await self._diag.record(
|
||||
stage="step3_dashboard", level="warning", event="dashboard_json_retry",
|
||||
message=f"大屏数据第 {attempt + 1} 次抽取失败(缺 report-json / JSON 非法),已追加纠偏提示重试",
|
||||
detail={"thread_id": thread_id, "attempt": attempt + 1},
|
||||
agent_name=DASHBOARD_AGENT_ID,
|
||||
)
|
||||
return None
|
||||
Reference in new issue
Block a user