feat(roundtable): 会商诊断日志 + 后台挂起卡死兜底(idle 超时)
- 新增会商全流程诊断日志:新表 roundtable_diagnostics + 仓储 + admin API (/api/roundtable-diagnostics) + 管理员诊断页(会商页右上角设置按钮入口), 覆盖第一/二/三步及后台执行器/进程内网关各处报错与关键事件,支持按 范围/阶段/级别/事件/关键词筛选 + 分页。诊断写入全程 best-effort,绝不打断会商。 - 修复后台挂起「执行完第一个席位后永久卡死」:进程内网关每轮 agent run 改为 idle(无活动)超时(默认 20min)——只要还在持续产出 token/工具事件就一直等, 仅当连续无任何输出才判卡死并取消+报错+落 seat_turn_timeout 诊断;慢但活着的 生成绝不打断。可经 ROUNDTABLE_SEAT_IDLE_TIMEOUT_SECONDS 调整。 - 一并提交进行中的会商 seat-skill 注入 / 业务链条编辑器相关改动。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8ca16c74bc
commit
4a403c95c5
27 files changed
+1873
-61
No files matched your search
+112
@@ -0,0 +1,112 @@
|
||||
"""圆桌会商诊断日志持久化。
|
||||
|
||||
对外暴露:
|
||||
- ``make_roundtable_diagnostic_store(sf)`` —— 建仓储(deps.py 启动时调)。
|
||||
- ``register_default_store`` / ``get_default_store`` —— 进程级默认仓储(跨 app/harness 边界,
|
||||
与 ``roundtable_jobs`` 同套做法:harness 侧编排引擎拿不到 FastAPI ``app.state``)。
|
||||
- ``record_diagnostic(...)`` —— **best-effort** 写一条事件(store 为 None / 出错均静默吞掉,
|
||||
诊断日志本身绝不能反过来打断会商)。
|
||||
- ``DiagnosticsRecorder`` —— 绑定一次作业上下文(scope/job_id/draft_id/task_id/user_id)的
|
||||
便捷记录器,引擎 / 网关 / 执行器逐处调用 ``await recorder.record(stage=..., level=..., ...)``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.roundtable_diagnostics.model import RoundtableDiagnosticRow
|
||||
from deerflow.persistence.roundtable_diagnostics.sql import RoundtableDiagnosticRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def make_roundtable_diagnostic_store(session_factory) -> RoundtableDiagnosticRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return RoundtableDiagnosticRepository(session_factory)
|
||||
|
||||
|
||||
# ── 进程级默认仓储(跨 app/harness 边界)──────────────────────────────────────────
|
||||
_default_store: RoundtableDiagnosticRepository | None = None
|
||||
|
||||
|
||||
def register_default_store(store: RoundtableDiagnosticRepository | None) -> None:
|
||||
global _default_store
|
||||
_default_store = store
|
||||
|
||||
|
||||
def get_default_store() -> RoundtableDiagnosticRepository | None:
|
||||
return _default_store
|
||||
|
||||
|
||||
async def record_diagnostic(store: RoundtableDiagnosticRepository | None = None, **fields: Any) -> None:
|
||||
"""Best-effort 写一条诊断事件。``store`` 缺省用进程级默认仓储。任何异常静默吞掉。"""
|
||||
repo = store or _default_store
|
||||
if repo is None:
|
||||
return
|
||||
try:
|
||||
await repo.record(**fields)
|
||||
except Exception: # noqa: BLE001 — 诊断日志写失败绝不打断会商
|
||||
logger.debug("roundtable diagnostic record failed", exc_info=True)
|
||||
|
||||
|
||||
class DiagnosticsRecorder:
|
||||
"""绑定一次会商上下文的记录器。所有 ``record`` 均 best-effort。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: RoundtableDiagnosticRepository | None = None,
|
||||
*,
|
||||
scope: str = "background",
|
||||
job_id: str | None = None,
|
||||
draft_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._scope = scope
|
||||
self._job_id = job_id
|
||||
self._draft_id = draft_id
|
||||
self._task_id = task_id
|
||||
self._user_id = user_id
|
||||
|
||||
async def record(
|
||||
self,
|
||||
*,
|
||||
stage: str,
|
||||
level: str,
|
||||
event: str,
|
||||
message: str | None = None,
|
||||
detail: Any = None,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
cycle: int | None = None,
|
||||
) -> None:
|
||||
await record_diagnostic(
|
||||
self._store,
|
||||
scope=self._scope,
|
||||
stage=stage,
|
||||
level=level,
|
||||
event=event,
|
||||
message=message,
|
||||
detail=detail,
|
||||
job_id=self._job_id,
|
||||
draft_id=self._draft_id,
|
||||
task_id=self._task_id,
|
||||
user_id=self._user_id,
|
||||
agent_id=agent_id,
|
||||
agent_name=agent_name,
|
||||
cycle=cycle,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RoundtableDiagnosticRow",
|
||||
"RoundtableDiagnosticRepository",
|
||||
"DiagnosticsRecorder",
|
||||
"make_roundtable_diagnostic_store",
|
||||
"register_default_store",
|
||||
"get_default_store",
|
||||
"record_diagnostic",
|
||||
]
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
"""圆桌会商「诊断日志」ORM 模型。
|
||||
|
||||
记录会商全流程(第一步意图 / 第二步研讨 / 第三步报告 + 后台挂起执行器 / 进程内网关)
|
||||
各处可能发生的报错与关键事件,供管理员在前端诊断页筛选排查。
|
||||
|
||||
设计要点:
|
||||
- **全局/共享**:不按 user 分权(``user_id`` 仅审计),管理员读全量;写入开放(best-effort)。
|
||||
- 一行 = 一条事件。``detail`` 用 ``PortableLongText`` 存结构化补充信息(堆栈尾、上下文)。
|
||||
- 按 ``created_at`` / ``stage`` / ``level`` / ``job_id`` / ``task_id`` 建索引,支撑筛选。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class RoundtableDiagnosticRow(Base):
|
||||
__tablename__ = "roundtable_diagnostics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# background(后台挂起)| foreground(前台实时会商)。
|
||||
scope: Mapped[str] = mapped_column(String(16), nullable=False, default="background", index=True)
|
||||
# 阶段:step1_intent / step2_init / step2_leader / step2_seat / step2_orchestration /
|
||||
# step3_summary / step3_report / step3_dashboard / step3_structure / upstream / job。
|
||||
stage: Mapped[str] = mapped_column(String(32), nullable=False, default="job", index=True)
|
||||
# 级别:info | warning | error。
|
||||
level: Mapped[str] = mapped_column(String(16), nullable=False, default="info", index=True)
|
||||
# 短事件码(如 seat_turn_timeout / model_error / report_truncated / dispatch_invalid)。
|
||||
event: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
# 人类可读的报错说明(中文)。
|
||||
message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# 结构化补充:堆栈尾、上下文、原始 body 片段等。
|
||||
detail: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
# 关联上下文(均可空)。
|
||||
job_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
draft_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
cycle: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), index=True
|
||||
)
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
"""SQLAlchemy-backed 圆桌会商诊断日志 store。
|
||||
|
||||
写入(``record``)best-effort、全局共享;读取(``list_logs`` / ``count`` / ``facets``)
|
||||
带多维筛选,供管理员诊断页用。``detail`` 以 JSON 序列化进 ``PortableLongText``。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.roundtable_diagnostics.model import RoundtableDiagnosticRow
|
||||
|
||||
|
||||
def _dumps(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return value or None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return value # 非 JSON 文本(如纯字符串 detail)原样返回
|
||||
|
||||
|
||||
class RoundtableDiagnosticRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _to_dict(row: RoundtableDiagnosticRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"scope": row.scope,
|
||||
"stage": row.stage,
|
||||
"level": row.level,
|
||||
"event": row.event,
|
||||
"message": row.message,
|
||||
"detail": _loads(row.detail),
|
||||
"job_id": row.job_id,
|
||||
"draft_id": row.draft_id,
|
||||
"task_id": row.task_id,
|
||||
"user_id": row.user_id,
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"cycle": row.cycle,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
}
|
||||
|
||||
async def record(
|
||||
self,
|
||||
*,
|
||||
scope: str = "background",
|
||||
stage: str = "job",
|
||||
level: str = "info",
|
||||
event: str = "",
|
||||
message: str | None = None,
|
||||
detail: Any = None,
|
||||
job_id: str | None = None,
|
||||
draft_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
agent_id: str | None = None,
|
||||
agent_name: str | None = None,
|
||||
cycle: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""插入一条诊断事件,返回序列化 dict。"""
|
||||
row = RoundtableDiagnosticRow(
|
||||
id=uuid4().hex,
|
||||
scope=(scope or "background")[:16],
|
||||
stage=(stage or "job")[:32],
|
||||
level=(level or "info")[:16],
|
||||
event=(event or "")[:64],
|
||||
message=message,
|
||||
detail=_dumps(detail),
|
||||
job_id=job_id,
|
||||
draft_id=draft_id,
|
||||
task_id=task_id,
|
||||
user_id=user_id,
|
||||
agent_id=(agent_id or None) if agent_id is None else str(agent_id)[:128],
|
||||
agent_name=(agent_name or None) if agent_name is None else str(agent_name)[:255],
|
||||
cycle=cycle,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._to_dict(row)
|
||||
|
||||
def _apply_filters(
|
||||
self,
|
||||
stmt,
|
||||
*,
|
||||
scope: str | None,
|
||||
stage: str | None,
|
||||
level: str | None,
|
||||
event: str | None,
|
||||
job_id: str | None,
|
||||
draft_id: str | None,
|
||||
task_id: str | None,
|
||||
user_id: str | None,
|
||||
q: str | None,
|
||||
since: datetime | None,
|
||||
until: datetime | None,
|
||||
):
|
||||
R = RoundtableDiagnosticRow
|
||||
if scope:
|
||||
stmt = stmt.where(R.scope == scope)
|
||||
if stage:
|
||||
stmt = stmt.where(R.stage == stage)
|
||||
if level:
|
||||
stmt = stmt.where(R.level == level)
|
||||
if event:
|
||||
stmt = stmt.where(R.event == event)
|
||||
if job_id:
|
||||
stmt = stmt.where(R.job_id == job_id)
|
||||
if draft_id:
|
||||
stmt = stmt.where(R.draft_id == draft_id)
|
||||
if task_id:
|
||||
stmt = stmt.where(R.task_id == task_id)
|
||||
if user_id:
|
||||
stmt = stmt.where(R.user_id == user_id)
|
||||
if q:
|
||||
like = f"%{q}%"
|
||||
stmt = stmt.where(R.message.ilike(like))
|
||||
if since is not None:
|
||||
stmt = stmt.where(R.created_at >= since)
|
||||
if until is not None:
|
||||
stmt = stmt.where(R.created_at <= until)
|
||||
return stmt
|
||||
|
||||
async def list_logs(
|
||||
self,
|
||||
*,
|
||||
scope: str | None = None,
|
||||
stage: str | None = None,
|
||||
level: str | None = None,
|
||||
event: str | None = None,
|
||||
job_id: str | None = None,
|
||||
draft_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
user_id: str | None = None,
|
||||
q: str | None = None,
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""筛选 + 分页(newest-first)。返回 ``(items, total)``。"""
|
||||
R = RoundtableDiagnosticRow
|
||||
base = self._apply_filters(
|
||||
select(R),
|
||||
scope=scope, stage=stage, level=level, event=event,
|
||||
job_id=job_id, draft_id=draft_id, task_id=task_id, user_id=user_id,
|
||||
q=q, since=since, until=until,
|
||||
)
|
||||
count_stmt = self._apply_filters(
|
||||
select(func.count()).select_from(R),
|
||||
scope=scope, stage=stage, level=level, event=event,
|
||||
job_id=job_id, draft_id=draft_id, task_id=task_id, user_id=user_id,
|
||||
q=q, since=since, until=until,
|
||||
)
|
||||
page_stmt = base.order_by(R.created_at.desc(), R.id.desc()).limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
total = int((await session.execute(count_stmt)).scalar() or 0)
|
||||
rows = (await session.execute(page_stmt)).scalars().all()
|
||||
return [self._to_dict(r) for r in rows], total
|
||||
|
||||
async def facets(self) -> dict[str, list[str]]:
|
||||
"""返回各筛选维度的 distinct 取值(供前端下拉),供筛选 UI 用。"""
|
||||
R = RoundtableDiagnosticRow
|
||||
out: dict[str, list[str]] = {}
|
||||
async with self._sf() as session:
|
||||
for key, col in (("scope", R.scope), ("stage", R.stage), ("level", R.level), ("event", R.event)):
|
||||
rows = (await session.execute(select(col).distinct())).scalars().all()
|
||||
out[key] = sorted([str(v) for v in rows if v])
|
||||
return out
|
||||
|
||||
async def list_older_than(self, cutoff: datetime, *, limit: int = 1000) -> list[str]:
|
||||
R = RoundtableDiagnosticRow
|
||||
stmt = select(R.id).where(R.created_at < cutoff).limit(limit)
|
||||
async with self._sf() as session:
|
||||
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def delete_by_ids(self, ids: list[str]) -> int:
|
||||
if not ids:
|
||||
return 0
|
||||
R = RoundtableDiagnosticRow
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(R).where(R.id.in_(ids)))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
Reference in new issue
Block a user