feat(roundtable): 会商诊断日志 + 后台挂起卡死兜底(idle 超时)
- 新增会商全流程诊断日志:新表 roundtable_diagnostics + 仓储 + admin API (/api/roundtable-diagnostics) + 管理员诊断页(会商页右上角设置按钮入口), 覆盖第一/二/三步及后台执行器/进程内网关各处报错与关键事件,支持按 范围/阶段/级别/事件/关键词筛选 + 分页。诊断写入全程 best-effort,绝不打断会商。 - 修复后台挂起「执行完第一个席位后永久卡死」:进程内网关每轮 agent run 改为 idle(无活动)超时(默认 20min)——只要还在持续产出 token/工具事件就一直等, 仅当连续无任何输出才判卡死并取消+报错+落 seat_turn_timeout 诊断;慢但活着的 生成绝不打断。可经 ROUNDTABLE_SEAT_IDLE_TIMEOUT_SECONDS 调整。 - 一并提交进行中的会商 seat-skill 注入 / 业务链条编辑器相关改动。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8ca16c74bc
commit
4a403c95c5
27 files changed
+1873
-61
No files matched your search
@@ -62,7 +62,9 @@ from fastapi import APIRouter, Body, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
|
||||
from app.gateway.roundtable_diag import record_foreground_diag
|
||||
from app.gateway.roundtable_run_policy import merge_skill_stop_names, roundtable_run_policy
|
||||
from app.gateway.roundtable_seat_skills import append_seat_skill_directive
|
||||
from app.gateway.routers._roundtable_seed import (
|
||||
COORDINATOR_AGENT_ID,
|
||||
DASHBOARD_AGENT_ID,
|
||||
@@ -80,6 +82,23 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/multi-agent", tags=["multi-agent"])
|
||||
|
||||
|
||||
def _stage_for_agent(agent_type: str, agent_name: str) -> str:
|
||||
"""把一次 /run/stream 的 (agent_type, agent_name) 映射成诊断日志的 stage。
|
||||
|
||||
Step3 的四个内置单例(report/summary/dashboard/structure)也走 special 路径,
|
||||
据 agent_name 归到对应 step3_* 阶段;其余 leader→step2_leader、special→step2_seat。
|
||||
"""
|
||||
if agent_name == REPORT_AGENT_ID:
|
||||
return "step3_report"
|
||||
if agent_name == SUMMARY_AGENT_ID:
|
||||
return "step3_summary"
|
||||
if agent_name == DASHBOARD_AGENT_ID:
|
||||
return "step3_dashboard"
|
||||
if agent_name == STRUCTURE_AGENT_ID:
|
||||
return "step3_structure"
|
||||
return "step2_leader" if agent_type == "leader" else "step2_seat"
|
||||
|
||||
|
||||
# 圆桌(intent / recommend / 席位 / 协调)创建的 thread 都是功能型内部线程,
|
||||
# 不是用户的普通对话。打上这个 metadata,前端「最近的对话」列表(recent-chat-list.tsx)
|
||||
# 据 `metadata.thread_type` 过滤掉,与 scheduler / bootstrap 系统会话同一套机制。
|
||||
@@ -892,9 +911,19 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
|
||||
})
|
||||
except HTTPException as exc:
|
||||
logger.warning("multi-agent init HTTPException: %s %s", exc.status_code, exc.detail)
|
||||
await record_foreground_diag(
|
||||
request, stage="step2_init", level="error", event="init_http_error",
|
||||
message=f"会商初始化失败 [{exc.status_code}]:{exc.detail}",
|
||||
detail={"status_code": exc.status_code, "detail": str(exc.detail)},
|
||||
)
|
||||
yield _frame({"event": "error", "detail": f"[{exc.status_code}] {exc.detail}"})
|
||||
except Exception as exc:
|
||||
logger.exception("multi-agent init unexpected failure")
|
||||
await record_foreground_diag(
|
||||
request, stage="step2_init", level="error", event="init_unexpected",
|
||||
message=f"会商初始化未预期异常:{exc!r}",
|
||||
detail={"type": type(exc).__name__, "error": str(exc)},
|
||||
)
|
||||
yield _frame({"event": "error", "detail": f"unexpected: {exc!r}"})
|
||||
|
||||
return StreamingResponse(
|
||||
@@ -922,8 +951,16 @@ async def init_report_thread(request: Request) -> dict[str, str]:
|
||||
``agent_name="roundtable-report"``、``d_agent_thread_id={REPORT_AGENT_ID: thread_id}``)
|
||||
跑这一轮,智能体把图表 HTML 写进该线程的 ``outputs/``,再由沙箱预览。
|
||||
"""
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
try:
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
except Exception as exc:
|
||||
await record_foreground_diag(
|
||||
request, stage="step3_report", level="error", event="thread_init_failed",
|
||||
message=f"创建结果绘制线程失败:{exc}", detail={"type": type(exc).__name__, "error": str(exc)},
|
||||
agent_id=REPORT_AGENT_ID,
|
||||
)
|
||||
raise
|
||||
return {"agent_name": REPORT_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
@@ -942,8 +979,16 @@ async def init_summary_thread(request: Request) -> dict[str, str]:
|
||||
跑这一轮(后续问答/增量改**复用同一线程**,不重新 init),智能体把 Markdown 报告
|
||||
写进该线程的 ``outputs/方案总结报告.md``,再由沙箱预览。
|
||||
"""
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
try:
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
except Exception as exc:
|
||||
await record_foreground_diag(
|
||||
request, stage="step3_summary", level="error", event="thread_init_failed",
|
||||
message=f"创建总结报告线程失败:{exc}", detail={"type": type(exc).__name__, "error": str(exc)},
|
||||
agent_id=SUMMARY_AGENT_ID,
|
||||
)
|
||||
raise
|
||||
return {"agent_name": SUMMARY_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
@@ -964,8 +1009,16 @@ async def init_dashboard_thread(request: Request) -> dict[str, str]:
|
||||
跑这一轮(后续问答/增量改**复用同一线程**,不重新 init),智能体在对话里输出
|
||||
```report-json,前端用 ``extractReportData`` 抽取后按当前主题渲染大屏。
|
||||
"""
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
try:
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
except Exception as exc:
|
||||
await record_foreground_diag(
|
||||
request, stage="step3_dashboard", level="error", event="thread_init_failed",
|
||||
message=f"创建大屏数据线程失败:{exc}", detail={"type": type(exc).__name__, "error": str(exc)},
|
||||
agent_id=DASHBOARD_AGENT_ID,
|
||||
)
|
||||
raise
|
||||
return {"agent_name": DASHBOARD_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
@@ -986,8 +1039,16 @@ async def init_structure_thread(request: Request) -> dict[str, str]:
|
||||
跑这一轮,把 taskId + 总结报告 + 结构化流程图交给它,由它挑选合适的入库技能(名称
|
||||
可能含「六步法」/自行判断适用性)读 SKILL.md 并按其契约调用完成入库。
|
||||
"""
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
try:
|
||||
ensure_roundtable_functional_agents()
|
||||
thread_id = await _create_thread_direct(request, metadata=_ROUNDTABLE_THREAD_METADATA)
|
||||
except Exception as exc:
|
||||
await record_foreground_diag(
|
||||
request, stage="step3_structure", level="error", event="thread_init_failed",
|
||||
message=f"创建数据入库线程失败:{exc}", detail={"type": type(exc).__name__, "error": str(exc)},
|
||||
agent_id=STRUCTURE_AGENT_ID,
|
||||
)
|
||||
raise
|
||||
return {"agent_name": STRUCTURE_AGENT_ID, "thread_id": thread_id}
|
||||
|
||||
|
||||
@@ -995,6 +1056,16 @@ async def init_structure_thread(request: Request) -> dict[str, str]:
|
||||
# 上游 LangGraph SSE 透传(intent / recommend / leader / special 共用)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 回环 /runs/stream 在**响应头阶段**返回这些状态码时,视为「单 worker + 单 sqlite 锁被
|
||||
# 并发打满」造成的瞬时失败,退避重试(详见 _stream_upstream 内注释):
|
||||
# 409 同 thread 抢占竞态;429 限流;500/503/504 高并发下 start_run 里 DB/checkpoint 写抖动。
|
||||
# 不含 400/401/403/404/422 等确定性错误——那些重试也没用,立即上报。
|
||||
_UPSTREAM_RETRYABLE_STATUSES = frozenset({409, 429, 500, 502, 503, 504})
|
||||
# 重试次数(不含首次)。总尝试 = 1 + _UPSTREAM_MAX_RETRIES。
|
||||
_UPSTREAM_MAX_RETRIES = 2
|
||||
# 指数退避基数(秒):0.4 → 0.8 → …,足够让在跑的席位 run 让出 sqlite 锁,又不至于明显拖慢。
|
||||
_UPSTREAM_RETRY_BASE_DELAY = 0.4
|
||||
|
||||
|
||||
async def _stream_upstream(
|
||||
client: httpx.AsyncClient,
|
||||
@@ -1017,36 +1088,74 @@ async def _stream_upstream(
|
||||
|
||||
url = f"{base}/api/threads/{thread_id}/runs/stream"
|
||||
captured: list[str] = []
|
||||
t_send = time.perf_counter()
|
||||
async with client.stream("POST", url, json=body, headers=headers) as resp:
|
||||
t_headers = time.perf_counter()
|
||||
if resp.status_code != 200:
|
||||
body_text = (await resp.aread()).decode("utf-8", "replace")[:1000]
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"upstream {url} returned HTTP {resp.status_code}: {body_text}",
|
||||
)
|
||||
agent_for_log = (body.get("context") or {}).get("agent_name") or thread_id
|
||||
logger.info(
|
||||
"[upstream-timing] %s headers_in=%.3fs",
|
||||
agent_for_log,
|
||||
t_headers - t_send,
|
||||
)
|
||||
first_line = True
|
||||
async for line in resp.aiter_lines():
|
||||
if not line:
|
||||
continue
|
||||
if first_line:
|
||||
first_line = False
|
||||
logger.info(
|
||||
"[upstream-timing] %s first_line=%.3fs (LangGraph runtime startup + LLM first byte)",
|
||||
agent_for_log,
|
||||
time.perf_counter() - t_headers,
|
||||
agent_for_log = (body.get("context") or {}).get("agent_name") or thread_id
|
||||
|
||||
# 头阶段非 200 的瞬时重试。
|
||||
# 背景:网关是**单 worker** + checkpointer 是**单连接 aiosqlite**(全进程一把
|
||||
# asyncio.Lock 串行 checkpoint 读写)。圆桌第三步进入时,前端把「方案总结/结果绘制/
|
||||
# 大屏」三个重 run **同一刻并行**打出来,叠加第二步 on_disconnect=continue 仍在后台
|
||||
# 跑的席位 run,瞬时把单锁/单 worker 打满 —— 此时回环子请求 /runs/stream 的 start_run
|
||||
# 里那串 DB / checkpoint 写有概率抛错(或同 thread 抢占竞态),返回非 200,被本函数原样
|
||||
# 包成 502 漏给前端(用户侧表现为"进第三步概率性 502")。
|
||||
# 关键安全前提:这个非 200 发生在**响应头阶段**,本函数**尚未 yield 任何 SSE 行**给下游,
|
||||
# 所以"换条连接重发"是幂等安全的(不会重复透传半截流)。对可恢复的瞬时状态码退避重试几次,
|
||||
# 把绝大多数并发尖峰造成的偶发失败吃掉;非可恢复状态码(如 400/401/403/404)立即按原逻辑
|
||||
# 报错,不浪费重试。一旦进入流式(已 yield 行)就绝不重试。
|
||||
last_status = 0
|
||||
last_body = ""
|
||||
for attempt in range(_UPSTREAM_MAX_RETRIES + 1):
|
||||
t_send = time.perf_counter()
|
||||
async with client.stream("POST", url, json=body, headers=headers) as resp:
|
||||
t_headers = time.perf_counter()
|
||||
if resp.status_code != 200:
|
||||
last_status = resp.status_code
|
||||
last_body = (await resp.aread()).decode("utf-8", "replace")[:1000]
|
||||
if resp.status_code in _UPSTREAM_RETRYABLE_STATUSES and attempt < _UPSTREAM_MAX_RETRIES:
|
||||
delay = _UPSTREAM_RETRY_BASE_DELAY * (2**attempt)
|
||||
logger.warning(
|
||||
"[upstream-retry] %s 回环 %s 返回 HTTP %s,%.2fs 后重试(第 %d/%d 次):%s",
|
||||
agent_for_log,
|
||||
url,
|
||||
resp.status_code,
|
||||
delay,
|
||||
attempt + 1,
|
||||
_UPSTREAM_MAX_RETRIES,
|
||||
last_body[:200],
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
continue # 重连:换条新连接重发同一 body(尚未 yield 任何行,幂等安全)
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"upstream {url} returned HTTP {resp.status_code}: {last_body}",
|
||||
)
|
||||
yield line, None
|
||||
if line.startswith("data:"):
|
||||
captured.append(line)
|
||||
yield None, captured
|
||||
logger.info(
|
||||
"[upstream-timing] %s headers_in=%.3fs%s",
|
||||
agent_for_log,
|
||||
t_headers - t_send,
|
||||
f" (retried x{attempt})" if attempt else "",
|
||||
)
|
||||
first_line = True
|
||||
async for line in resp.aiter_lines():
|
||||
if not line:
|
||||
continue
|
||||
if first_line:
|
||||
first_line = False
|
||||
logger.info(
|
||||
"[upstream-timing] %s first_line=%.3fs (LangGraph runtime startup + LLM first byte)",
|
||||
agent_for_log,
|
||||
time.perf_counter() - t_headers,
|
||||
)
|
||||
yield line, None
|
||||
if line.startswith("data:"):
|
||||
captured.append(line)
|
||||
yield None, captured
|
||||
return
|
||||
|
||||
# 理论上不可达(循环要么 return、要么 raise);留一条兜底防御。
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"upstream {url} returned HTTP {last_status}: {last_body}",
|
||||
)
|
||||
|
||||
|
||||
def _parse_last_messages(captured: list[str]) -> list[dict[str, Any]] | None:
|
||||
@@ -1677,11 +1786,18 @@ async def _special_run(
|
||||
if files:
|
||||
_read_tools = {"read_file", "ls", "grep", "view_image"}
|
||||
seat_excluded = [t for t in seat_excluded if t not in _read_tools]
|
||||
# 弱模型补偿:仅对真正的研讨席位(seat / seat_parallel;report/summary/dashboard/ingest
|
||||
# 等 Step3 单例不在此列)把其配置技能(名称 + SKILL.md 路径 + 必须先 read_file 再按其
|
||||
# 流程取信息的硬指令)显式拼进本轮任务,避免弱模型无视系统提示里的技能块(见
|
||||
# roundtable_seat_skills 模块说明)。无配置技能时原样返回。
|
||||
seat_message = new_message
|
||||
if role in ("seat", "seat_parallel"):
|
||||
seat_message = append_seat_skill_directive(agent_name, new_message)
|
||||
body = _run_payload(
|
||||
agent_name=agent_name,
|
||||
model_name=model_name,
|
||||
thread_id=thread_id,
|
||||
new_message=new_message,
|
||||
new_message=seat_message,
|
||||
excluded_tools=seat_excluded,
|
||||
skill_stop_names=merge_skill_stop_names(policy.skill_stop_names, skill_stop_names),
|
||||
# 席位执行模式覆盖 policy 的 thinking/reasoning(None → 回退 policy 默认);
|
||||
@@ -1899,6 +2015,13 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
|
||||
yield chunk if chunk.endswith("\n") else chunk + "\n"
|
||||
except HTTPException as exc:
|
||||
logger.warning("multi-agent stream HTTPException: %s %s", exc.status_code, exc.detail)
|
||||
await record_foreground_diag(
|
||||
request, stage=_stage_for_agent(agent_type, agent_name), level="error",
|
||||
event="run_http_error",
|
||||
message=f"「{agent_name}」运行失败 [{exc.status_code}]:{exc.detail}",
|
||||
detail={"status_code": exc.status_code, "detail": str(exc.detail), "agent_type": agent_type},
|
||||
agent_id=agent_name, agent_name=display_name or agent_name,
|
||||
)
|
||||
err_frame = {
|
||||
"status": "error",
|
||||
"content": f"[{exc.status_code}] {exc.detail}",
|
||||
@@ -1907,6 +2030,13 @@ async def run_multi_agent_stream(request: Request, payload: dict = Body(...)) ->
|
||||
yield f"data: {json.dumps(err_frame, ensure_ascii=False)}\n\n"
|
||||
except Exception as exc:
|
||||
logger.exception("multi-agent stream unexpected failure")
|
||||
await record_foreground_diag(
|
||||
request, stage=_stage_for_agent(agent_type, agent_name), level="error",
|
||||
event="run_unexpected",
|
||||
message=f"「{agent_name}」运行未预期异常:{exc!r}",
|
||||
detail={"type": type(exc).__name__, "error": str(exc), "agent_type": agent_type},
|
||||
agent_id=agent_name, agent_name=display_name or agent_name,
|
||||
)
|
||||
err_frame = {
|
||||
"status": "error",
|
||||
"content": f"unexpected: {exc!r}",
|
||||
|
||||
Reference in new issue
Block a user