初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,723 @@
"""圆桌后台作业执行器(Phase 3a)。
把 Phase 2 的编排引擎(``run_orchestration``)作为**后台 asyncio 任务**跑起来,
survive 发起请求的生命周期。每个任务:
1. ``set_current_user``:用作业的 user_id 设进程内用户上下文,让引擎里的进度落库
(``JobProgressWriter`` → ``RoundtableJobRepository.update_progress(user_id=AUTO)``)
正确归属(与 scheduled task 后台执行同一套做法);
2. 用 ``gateway_factory`` 构造一个 ``RoundtableGateway``(Phase 3a 默认模拟网关,
Phase 3b 换成进程内真网关),跑完整编排(循环 → 共识 → 报告)。
任务句柄登记在 ``_tasks`` 里,供 ``cancel`` 用;完成后自动摘除。
"""
from __future__ import annotations
import asyncio
import logging
import os
import uuid as _uuid
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.agents.roundtable_orchestrator import (
JobProgressWriter,
SeatRef,
run_orchestration,
)
from deerflow.persistence.roundtable_diagnostics import DiagnosticsRecorder, get_default_store
from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError
from deerflow.runtime.user_context import reset_current_user, set_current_user
logger = logging.getLogger(__name__)
# 本 worker 进程的唯一标识(Phase 3 租约持有者)。pid 保证同机多 worker 进程互不相同,
# 追加随机后缀避免 pid 复用导致的极端撞车。跨进程「谁在跑这个作业」由它写进 lease_owner。
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
# 租约 TTL 与心跳间隔(Phase 3)。心跳每 _HEARTBEAT_INTERVAL 秒续租一次,把 lease_until
# 推到 now + _LEASE_TTL_SECONDS。只要 worker 活着,租约永远不过期、别的 worker 抢不走;
# worker 崩溃后心跳停止,最长 _LEASE_TTL_SECONDS 后租约过期,dispatcher 在其它 worker 上
# 重新领取(attempt+1)—— 崩溃可接管的一致性来自 DB 租约而非进程内存。
# 约束:_LEASE_TTL_SECONDS 必须明显大于 _HEARTBEAT_INTERVAL(留足续租余量)。
_LEASE_TTL_SECONDS = 60
_HEARTBEAT_INTERVAL = 20.0
# 跨进程取消轮询间隔(秒)。cancel 端点把 ``status='cancel_requested'`` 写进 DB(权威意图),
# 实际跑作业的 worker 上的看门狗协程定期读 DB 检测到后,收口 ``cancelled`` 终态并对本进程的
# 编排 task 调 ``cancel()``。多 worker 下 cancel 请求可能打到非执行 worker,靠此机制最长延迟
# 一个轮询周期即可让幽灵作业停下。
_CANCEL_POLL_INTERVAL = 5.0
# 草稿乐观锁重试次数。每次尝试都带 expected_version(冲突则重读重试);重试全部耗尽后
# **绝不退化为无版本检查的 last-write-wins 盲写**,而是给作业置 draft_persist_pending
# 标记,由 outbox 补偿器(Phase 3)投影结果到草稿——一致性来自 DB 标记而非运气。
_DRAFT_WRITE_RETRIES = 3
# 「后台挂起」的 Step3 自动产出(方案总结报告 md + 结果绘制 HTML)总开关。
# 历史:2026-06-05 因后台 start_run 路径报告不稳定临时禁用;2026-06-12 定位到根因是
# ``_enforce_agent_access`` 对进程内虚拟 request 的 ``request.state`` 裸访问(AttributeError
# 杀死编排,见 tests/test_roundtable_inprocess_request.py),修复后恢复开启。
# 现 Step2 收口后自动跑:① roundtable-summary 写「方案总结报告.md」② roundtable-report
# 画「方案总览.html」,两段故障隔离(engine._run_step3),产物按手动同款形状写回草稿。
ROUNDTABLE_BACKGROUND_REPORT_ENABLED = True
class _JobUser:
"""最小 CurrentUser:满足 user_context 的 ``.id`` 结构协议。"""
def __init__(self, user_id: str) -> None:
self.id = user_id
# 头像配色:与前端 getAvatarTypeFor 对齐(内置圆桌席位固定色,其余按序轮换)。
_BUILTIN_AVATAR = {
"roundtable-intelligence": "purple",
"roundtable-environment": "emerald",
"roundtable-solution-design": "human",
"roundtable-risk-review": "amber",
"roundtable-execution-plan": "pink",
"roundtable-summary": "blue",
}
_FALLBACK_AVATAR = ["blue", "purple", "emerald", "amber", "pink"]
def _avatar_for(agent_id: str, index: int) -> str:
return _BUILTIN_AVATAR.get(agent_id) or _FALLBACK_AVATAR[index % len(_FALLBACK_AVATAR)]
# 工具步骤卡的回退文案。前端 getStepDisplay 对 tool 步骤是从 key+args 推图标/文案的,
# 存的 label 基本不展示——这里给个合理回退即可(与前端默认文案对齐)。
_STEP_LABEL = {
"agent_orchestration": "派活给子智能体",
"write_file": "写入文件",
"str_replace": "写入文件",
"read_file": "读取文件",
"ls": "列出文件夹",
"bash": "执行命令",
"web_search": "搜索相关信息",
"ask_clarification": "需要你的协助",
"present_files": "展示文件",
}
def _steps_from_tool_calls(tool_calls: list[dict], dialogue_id: object, ts: str) -> list[dict]:
"""把一轮 tool_calls 映射成前端 ``Step2Dialogue.steps``(工具步骤卡)。
- ``key`` = ``tool_calling:<toolName>``——前端 ``getStepDisplay`` 据此推图标 + 文案;
- ``id`` = ``<dialogueId>-<i>``——对齐前端 ``${bubbleId}-${stepIndex}``;
- ``args`` 原样带上(write_file 的 path/content、派活的 agent_name/task 等)。
"""
steps: list[dict] = []
for i, tc in enumerate(tool_calls or []):
name = tc.get("name") or ""
if not name:
continue
steps.append(
{
"id": f"{dialogue_id}-{i}",
"key": f"tool_calling:{name}",
"label": _STEP_LABEL.get(name, f"使用 “{name}” 工具"),
"ts": ts,
"args": tc.get("args") or {},
}
)
return steps
def _seat_stub_messages(dialogues: list[dict]) -> dict[str, list[dict]]:
"""从各对话的 write_file 调用重建 ``seatStubMessages``(threadId → [stub ai message])。
对齐前端 ``upsertSeatToolCall`` 的 stub 形状与 id 约定(消息 ``roundtable-stub-<tid>``、
工具调用 ``tc-<tid>-<i>``),使加载草稿后点开产物的虚拟 URL 能正确解析到内容。
同一 thread 跨轮写多次按 path 去重(后写覆盖),与前端一致。
"""
by_tid: dict[str, dict[str, dict]] = {}
for d in dialogues:
tid = d.get("threadId")
if not tid:
continue
for tc in d.get("toolCalls") or []:
if tc.get("name") != "write_file":
continue
path = str((tc.get("args") or {}).get("path") or "")
by_tid.setdefault(tid, {})[path] = tc # 后写覆盖
out: dict[str, list[dict]] = {}
for tid, by_path in by_tid.items():
tool_calls = [
{"id": f"tc-{tid}-{i}", "name": "write_file", "args": tc.get("args") or {}}
for i, tc in enumerate(by_path.values())
]
out[tid] = [{"id": f"roundtable-stub-{tid}", "type": "ai", "content": "", "tool_calls": tool_calls}]
return out
def _to_step2_dialogue(d: dict, idx_by_agent: dict[str, int]) -> dict:
"""把作业的中性对话记录映射成前端 Step2Dialogue 形状(含 avatarType / steps /
agentThreadId / time),使写进草稿 step2.runs 后,前端 hydrateStep2Run 能像手动跑
一样直接渲染——步骤卡、时间戳、点开产物全部可用。"""
dialogue_id = d.get("id")
time_str = d.get("time") or ""
base: dict = {
"id": dialogue_id,
"content": d.get("content") or "",
"time": time_str,
"streaming": False,
"steps": _steps_from_tool_calls(d.get("toolCalls") or [], dialogue_id, time_str),
}
thread_id = d.get("threadId")
if thread_id:
base["agentThreadId"] = thread_id
if d.get("role") == "leader":
return {**base, "sender": "总控协调", "avatarType": "blue", "confidence": "派活说明"}
aid = d.get("agentId") or ""
return {
**base,
"sender": d.get("name") or aid,
"avatarType": _avatar_for(aid, idx_by_agent.get(aid, 0)),
"confidence": "子智能体交付",
}
@dataclass
class StartParams:
job_id: str
user_id: str
agents: list[SeatRef]
seed_message: str
model: str | None = None
mode: str = "recommend"
intent_text: str = ""
thread_ids: dict[str, str] | None = None
coordinator_name: str = "roundtable-coordinator"
# 关联草稿:作业完成后把 step3 报告写回该草稿,使「查看报告」可用。
draft_id: str | None = None
# 外部任务 id(taskId 深链会商)。非空 = task 作业:draft_id 指向独立的
# roundtable_task_drafts 表,研讨/报告写回 task-draft 存储(不分权)。
task_id: str | None = None
# 是否为「续跑」(resume 澄清)。首启=False → 真网关建全新线程(不复用前端线程,
# 避免与前端本地研讨的活跃 run 撞 409);续跑=True → 复用作业自己的线程。
is_resume: bool = False
# chain 模式来源链条(写进草稿 run 用,展示)。
chain: dict[str, Any] | None = None
# dag 模式分层编排计划(前端 OrchestrationPlan dict):驱动引擎 _run_dag + 写进草稿 run
# 供前端流程图渲染。None = 非 dag(引擎按 mode 回退 chain/recommend)。
orchestration_plan: dict[str, Any] | None = None
# 是否自动产出 Step3(方案总结报告 + 结果绘制)。默认沿用全局开关(现已恢复 True,
# 见 ROUNDTABLE_BACKGROUND_REPORT_ENABLED)。测试可显式覆盖。
enable_report: bool = ROUNDTABLE_BACKGROUND_REPORT_ENABLED
# 席位执行模式(快速/思考/专业问答/多智能体)派生参数,沿用 Step2 选择。仅 special 席位用:
# 覆盖 roundtable_run_policy("seat") 的 thinking/reasoning(None → policy 默认);
# subagent 仅 ultra 档为真。透传给进程内网关(InProcessRoundtableGateway)的席位 run。
seat_thinking_enabled: bool | None = None
seat_reasoning_effort: str | None = None
seat_subagent_enabled: bool = False
# 每席位「技能识别强化」开关映射 {agent_id: bool}(沿用 Step2 所选业务链条里每个席位的
# 配置)。透传给进程内网关,run_seat 按 agent_id 查表,与各席位 agent 自身 config.yaml 的
# 同名开关取 OR。默认空 dict。
seat_skill_directives: dict[str, bool] = field(default_factory=dict)
# 每席位「推理深度覆盖」映射 {agent_id: SeatMode}(值 flash/thinking/pro/ultra,沿用 Step2 所选
# 业务链条里每个席位单独配的推理深度)。透传给进程内网关,run_seat 按 agent_id 查表:命中 → 用该
# 档派生的 thinking/reasoning/subagent 覆盖作业级 seatMode;未命中 → 跟随作业级。默认空 dict。
seat_modes: dict[str, str] = field(default_factory=dict)
# 是否在研讨前先跑一轮**全席位并行『取数』**(业务链条逐条 opt-in):各席位并行只调技能取数、
# 不研讨,产出作为共享资料注入后续研讨轮。透传给 run_orchestration 的 gather_first。默认 False。
gather_first: bool = False
# 深链接业务码(rwfx→6BF 等)。透传给 run_orchestration → build_summary_prompt:写实业务额外注入
# 「业务链完整性核对」,让后台总结报告核对产出相对业务链完不完整。普通会商为空 → 不注入。
business_code: str | None = None
# Phase 3 租约持有者(= dispatcher 领取时写入的 worker_id)。执行器据此:① 所有进度落库
# 带 lease_owner(update_progress 租约门,非持租者的过期写被拒);② 心跳续租;③ 看门狗
# 检测到 cancel_requested 时以持租者身份收口 cancelled。None = 旧路径(不走租约门)。
lease_owner: str | None = None
# ── Phase 3:input_snapshot 序列化 / 反序列化 ─────────────────────────────────
#
# 路由 start/resume 时把整份启动入参序列化成 JSON 存进 ``input_snapshot``;dispatcher
# 在**任意 worker** 领取 queued/租约过期作业后,据此无损重建 StartParams —— 这是「跨进程
# 可接管 / 崩溃可恢复」的前提(不再依赖发起请求的那个 worker 的进程内存)。
#
# 快照 schema 版本 ``_SNAPSHOT_VERSION``:将来字段变更时升版并兼容读旧版。
_SNAPSHOT_VERSION = 1
def build_start_snapshot(
*,
user_id: str,
agents: list[SeatRef],
seed_message: str,
model: str | None = None,
mode: str = "recommend",
intent_text: str = "",
thread_ids: dict[str, str] | None = None,
coordinator_name: str = "roundtable-coordinator",
draft_id: str | None = None,
task_id: str | None = None,
is_resume: bool = False,
chain: dict[str, Any] | None = None,
orchestration_plan: dict[str, Any] | None = None,
enable_report: bool = ROUNDTABLE_BACKGROUND_REPORT_ENABLED,
seat_thinking_enabled: bool | None = None,
seat_reasoning_effort: str | None = None,
seat_subagent_enabled: bool = False,
seat_skill_directives: dict[str, bool] | None = None,
seat_modes: dict[str, str] | None = None,
gather_first: bool = False,
business_code: str | None = None,
) -> dict[str, Any]:
"""把一次启动入参序列化为可 JSON 落库的快照 dict(snake_case,对齐 StartParams)。"""
return {
"v": _SNAPSHOT_VERSION,
"user_id": user_id,
"agents": [{"agent_id": a.agent_id, "name": a.name} for a in agents],
"seed_message": seed_message,
"model": model,
"mode": mode,
"intent_text": intent_text,
"thread_ids": thread_ids,
"coordinator_name": coordinator_name,
"draft_id": draft_id,
"task_id": task_id,
"is_resume": is_resume,
"chain": chain,
"orchestration_plan": orchestration_plan,
"enable_report": enable_report,
"seat_thinking_enabled": seat_thinking_enabled,
"seat_reasoning_effort": seat_reasoning_effort,
"seat_subagent_enabled": seat_subagent_enabled,
"seat_skill_directives": dict(seat_skill_directives or {}),
"seat_modes": dict(seat_modes or {}),
"gather_first": bool(gather_first),
"business_code": business_code,
}
def params_from_snapshot(
snapshot: dict[str, Any] | None, *, job_id: str, lease_owner: str | None
) -> StartParams | None:
"""从 ``input_snapshot`` 无损重建 StartParams。快照缺失 / 损坏返回 None(调用方跳过)。"""
if not snapshot or not isinstance(snapshot, dict):
return None
try:
agents = [
SeatRef(str(a.get("agent_id") or ""), str(a.get("name") or a.get("agent_id") or ""))
for a in (snapshot.get("agents") or [])
if isinstance(a, dict) and a.get("agent_id")
]
if not agents:
return None
return StartParams(
job_id=job_id,
user_id=str(snapshot.get("user_id") or ""),
agents=agents,
seed_message=str(snapshot.get("seed_message") or ""),
model=snapshot.get("model"),
mode=str(snapshot.get("mode") or "recommend"),
intent_text=str(snapshot.get("intent_text") or ""),
thread_ids=snapshot.get("thread_ids"),
coordinator_name=str(snapshot.get("coordinator_name") or "roundtable-coordinator"),
draft_id=snapshot.get("draft_id"),
task_id=snapshot.get("task_id"),
is_resume=bool(snapshot.get("is_resume")),
chain=snapshot.get("chain"),
orchestration_plan=snapshot.get("orchestration_plan"),
enable_report=bool(snapshot.get("enable_report", ROUNDTABLE_BACKGROUND_REPORT_ENABLED)),
seat_thinking_enabled=snapshot.get("seat_thinking_enabled"),
seat_reasoning_effort=snapshot.get("seat_reasoning_effort"),
seat_subagent_enabled=bool(snapshot.get("seat_subagent_enabled")),
seat_skill_directives=dict(snapshot.get("seat_skill_directives") or {}),
seat_modes=dict(snapshot.get("seat_modes") or {}),
gather_first=bool(snapshot.get("gather_first")),
business_code=snapshot.get("business_code"),
lease_owner=lease_owner,
)
except Exception: # noqa: BLE001 — 快照损坏不应让 dispatcher 崩溃
logger.exception("failed to rebuild StartParams from snapshot for job %s", job_id)
return None
# 网关工厂:给定一次启动参数,返回一个 RoundtableGateway 实例。
GatewayFactory = Callable[[StartParams], object]
class RoundtableJobExecutor:
def __init__(self, store, *, gateway_factory: GatewayFactory, draft_store=None, task_draft_store=None) -> None:
self._store = store
self._gateway_factory = gateway_factory
# 草稿仓储(可选):作业完成后把 step3 写回草稿,供前端「查看报告」。
self._draft_store = draft_store
# task 草稿仓储(独立、不分权):task 作业(params.task_id 非空)写回这里而非个人草稿。
self._task_draft_store = task_draft_store
self._tasks: dict[str, asyncio.Task] = {}
def _draft_store_for(self, params: StartParams):
"""task 作业 → 独立 task-draft 存储;普通作业 → 个人草稿存储。"""
return self._task_draft_store if params.task_id else self._draft_store
def start_job(self, params: StartParams) -> bool:
"""启动一个后台编排任务。同一 job 已在跑则忽略(幂等)。返回是否新启动。"""
if params.job_id in self._tasks and not self._tasks[params.job_id].done():
return False
task = asyncio.create_task(self._run(params))
self._tasks[params.job_id] = task
task.add_done_callback(lambda t, jid=params.job_id: self._tasks.pop(jid, None))
return True
async def _run(self, params: StartParams) -> None:
token = set_current_user(_JobUser(params.user_id))
diag = DiagnosticsRecorder(
get_default_store(),
scope="background",
job_id=params.job_id,
draft_id=params.draft_id,
task_id=params.task_id,
user_id=params.user_id,
)
try:
await diag.record(
stage="job", level="info", event="job_started",
message=f"后台挂起作业启动(mode={params.mode}{',续跑' if params.is_resume else ''})",
detail={"mode": params.mode, "is_resume": params.is_resume, "agents": [a.agent_id for a in params.agents]},
)
gateway = self._gateway_factory(params)
thread_ids = params.thread_ids or {}
coordinator_name = params.coordinator_name
# 真网关需要先进程内建好 leader/seat 线程;模拟网关无 init_threads。
# 首启(非 resume)始终建**全新线程** —— 绝不复用前端本地研讨的线程
# (那些线程可能有活跃 run,复用会撞 409)。resume 时复用作业自己的线程。
init = getattr(gateway, "init_threads", None)
if init is not None and not params.is_resume:
thread_ids, coordinator_name = await init()
try:
await self._store.update_progress(
params.job_id,
lease_owner=params.lease_owner,
thread_ids=thread_ids,
coordinator_name=coordinator_name,
)
except Exception: # noqa: BLE001
logger.warning("persist thread_ids for job %s failed", params.job_id, exc_info=True)
# 每轮模型返回后,把对话作为一个带 jobId 的 run 写进草稿 step2.runs[]
# —— 和手动跑同一存取路径(draft_store.update_draft),加载草稿即可显示。
async def _on_dialogue(dialogues: list[dict], consensus: int) -> None:
await self._write_run_to_draft(
params, dialogues=dialogues, status="running", consensus=consensus, thread_ids=thread_ids
)
progress = JobProgressWriter(
self._store,
job_id=params.job_id,
agents=params.agents,
on_dialogue=_on_dialogue if (self._draft_store_for(params) and params.draft_id) else None,
recorder=diag,
# Phase 3:进度落库带租约持有者 —— update_progress 租约门只放行持租者的写,
# 非持租者(被接管/已取消)的过期进度写被 DB 条件 UPDATE 拒绝。
lease_owner=params.lease_owner,
)
# Phase 3 心跳续租:活着就持续把 lease_until 推后,保证本 worker 的租约不过期、
# 别的 worker 抢不走;本协程随编排 task 生死(finally 里 cancel)。崩溃则心跳停,
# 租约最长 _LEASE_TTL_SECONDS 后过期 → dispatcher 在其它 worker 接管。
main_task = asyncio.current_task()
heartbeat = (
asyncio.create_task(self._heartbeat(params.job_id, params.lease_owner, main_task))
if params.lease_owner
else None
)
# 跨进程取消看门狗:run_orchestration 是一个不透明长 await(多轮 LLM),
# 无法在其内部插检查点。改为起一个并行的看门狗协程,定期读 DB 的 job status;
# cancel 端点把 status=cancel_requested 写进 DB(权威意图),看门狗检测到后以**持租者**
# 身份收口 cancelled 终态(清唯一活跃键 + 租约)并对本编排 task 调 cancel() →
# CancelledError 在 run_orchestration 的下一个 await 点抛出并上抛。
# 这让 cancel 在多 worker 下跨进程生效——cancel 打到非执行 worker 也无妨。
watcher = asyncio.create_task(
self._watch_cancellation(params.job_id, main_task, params.lease_owner)
)
try:
result = await run_orchestration(
gateway=gateway,
progress=progress,
agents=params.agents,
thread_ids=thread_ids,
coordinator_name=coordinator_name,
model=params.model,
seed_message=params.seed_message,
mode=params.mode,
intent_text=params.intent_text,
enable_report=params.enable_report,
plan=params.orchestration_plan,
gather_first=params.gather_first,
business_code=params.business_code,
)
finally:
watcher.cancel()
with suppress(asyncio.CancelledError, Exception):
await watcher
if heartbeat is not None:
heartbeat.cancel()
with suppress(asyncio.CancelledError, Exception):
await heartbeat
# 终态:把最终对话 + step3 + jobStatus 落进同一个 run(含 furthest_step=3)。
await self._write_run_to_draft(
params,
dialogues=result.dialogues,
status=result.status,
consensus=100 if result.status == "done" else 95,
step3=result.step3,
thread_ids=thread_ids,
)
# step3 也挂 job(API / 弹窗用)。带租约门:仅持租者可写。
if result.status == "done" and result.step3:
try:
await self._store.update_progress(
params.job_id, lease_owner=params.lease_owner, step3=result.step3
)
except Exception: # noqa: BLE001
logger.warning("persist step3 to job %s failed", params.job_id, exc_info=True)
except asyncio.CancelledError:
# 干净停止即可,**不在被取消的任务里写 DB**:取消态下二次 await 会再抛
# CancelledError、并可能打断进行中的 sqlite 连接。``cancelled`` 终态由
# 取消看门狗(或租约过期后的 dispatcher 回收)权威写入。
logger.info("roundtable job %s cancelled", params.job_id)
raise
except Exception as exc: # noqa: BLE001 — 顶层兜底:run_orchestration 内部已尽量标 error
logger.exception("roundtable job %s crashed", params.job_id)
await diag.record(
stage="job", level="error", event="job_crashed",
message=f"后台挂起作业崩溃:{exc}",
detail={"type": type(exc).__name__, "error": str(exc)},
)
finally:
reset_current_user(token)
async def _heartbeat(
self, job_id: str, lease_owner: str, main_task: asyncio.Task
) -> None:
"""定期续租:把 ``lease_until`` 推到 now + TTL,声明「本 worker 仍在跑、仍持租」。
只要主任务活着且续租成功,别的 worker 的 dispatcher 看到租约未过期就不会抢;主任务
结束(或本协程被 cancel)后不再续租,租约自然到期。续租失败(租约已被接管 / 状态已变)
说明本 worker 已不是持租者,必须立即停止本地编排,避免过期 worker 继续消耗模型资源;
其任何迟到进度写也会被租约门拒绝。
"""
try:
while not main_task.done():
await asyncio.sleep(_HEARTBEAT_INTERVAL)
if main_task.done():
return
try:
ok = await self._store.renew_lease(
job_id,
lease_owner=lease_owner,
lease_until=datetime.now(UTC) + timedelta(seconds=_LEASE_TTL_SECONDS),
)
except Exception: # noqa: BLE001 — DB 抖动不中断心跳,下轮重试
logger.warning("roundtable job %s renew_lease failed", job_id, exc_info=True)
continue
if not ok:
logger.info(
"roundtable job %s: renew_lease rejected (lease lost or status changed)",
job_id,
)
if not main_task.done():
main_task.cancel()
return
except asyncio.CancelledError:
# 编排结束 → _run 的 finally cancel 了心跳 → 安静退出。
return
async def _watch_cancellation(
self, job_id: str, main_task: asyncio.Task, lease_owner: str | None
) -> None:
"""定期轮询 DB 检测跨进程取消标志(``cancel_requested``),并以持租者身份收口。
多 worker 下 cancel 请求可能打到**非执行** worker,该 worker 的 ``cancel()`` 在
``_tasks`` 里找不到此 job(返回 False)。但 cancel 端点始终把 ``status='cancel_requested'``
写进 DB(权威意图)。本看门狗跑在**实际执行作业的 worker** 上,检测到后:
1. 以**持租者**身份 ``finalize_cancel``——条件 UPDATE 把 cancel_requested 收口为终态
``cancelled``,同时清唯一活跃键 + 租约(一致性来自 DB,绝不双收口);
2. 对本地编排 task 调 ``cancel()``,让 ``run_orchestration`` 在下一个 await 点抛
``CancelledError`` 干净退出。
DB 读失败时静默重试(不中断看门狗)。兼容旧数据:若 DB 里已是终态 ``cancelled``
(旧版 cancel 端点直写),直接取消本地 task。
"""
try:
while not main_task.done():
await asyncio.sleep(_CANCEL_POLL_INTERVAL)
if main_task.done():
return
try:
# user_id=None → 不按 user 过滤,纯读 status(看门狗不关心归属)。
row = await self._store.get(job_id, user_id=None)
except Exception: # noqa: BLE001 — DB 读失败不中断看门狗,下轮重试
continue
status = (row.get("status") or "") if row is not None else ""
if status == "cancel_requested":
logger.info(
"roundtable job %s: detected cancel_requested in DB, finalizing "
"cancelled and cancelling orchestration task",
job_id,
)
# 以持租者身份收口;非持租者(理论上不该发生)则退化为无租约条件收口。
try:
await self._store.finalize_cancel(job_id, lease_owner=lease_owner)
except Exception: # noqa: BLE001 — 收口失败不阻断本地取消
logger.warning("roundtable job %s finalize_cancel failed", job_id, exc_info=True)
if not main_task.done():
main_task.cancel()
return
if status == "cancelled":
# 兼容:旧路径已直接把 cancelled 写进 DB —— 本地 task 照样停下。
logger.info(
"roundtable job %s: detected cancelled in DB, cancelling orchestration task",
job_id,
)
if not main_task.done():
main_task.cancel()
return
except asyncio.CancelledError:
# 编排正常结束 → _run 的 finally cancel 了看门狗 → 安静退出。
return
async def _write_run_to_draft(
self,
params: StartParams,
*,
dialogues: list[dict],
status: str,
consensus: int,
step3: dict | None = None,
thread_ids: dict[str, str] | None = None,
) -> None:
"""把本次后台作业作为一个带 jobId 的 run 写进草稿 step2.runs[](同手动跑存取)。
读现有 step2 → 按 run_id=``job-<jobId>`` upsert(保留其它手动 run)→ 设为 activeRun
→ update_draft。前端加载草稿即可像手动跑一样 hydrate 出对话。
带乐观锁重试:**每次**读-改-写都带上 ``expected_version``,若并发写导致版本冲突
(``DraftConcurrentWriteError``)则重新读取最新草稿、重新合并 run 再试。重试全部
耗尽后**绝不**退化为无版本检查的 last-write-wins 盲写,而是给作业置
``draft_persist_pending`` 标记(由 Phase 3 outbox 补偿器投影结果到草稿)——
消除后台作业与前台 PUT 并发时的 step2.runs 丢失更新,且一致性来自 DB 标记而非运气。
"""
draft_store = self._draft_store_for(params)
if draft_store is None or not params.draft_id:
return
# 这些只依赖 params/dialogues,重试间不变——提前算好。
idx_by_agent = {a.agent_id: i for i, a in enumerate(params.agents)}
s2_dialogues = [_to_step2_dialogue(d, idx_by_agent) for d in dialogues]
last_leader = next(
(d.get("content") for d in reversed(dialogues) if d.get("role") == "leader"), ""
)
run_id = f"job-{params.job_id}"
for attempt in range(_DRAFT_WRITE_RETRIES):
# 读最新草稿(冲突重试时重新读,拿到别的小伙伴刚写的 runs)。
try:
draft = await draft_store.get_draft(params.draft_id, params.user_id)
except Exception: # noqa: BLE001
draft = None
expected_version = (draft or {}).get("version")
step2 = dict((draft or {}).get("step2") or {})
runs = list(step2.get("runs") or [])
idx = next((i for i, r in enumerate(runs) if r.get("id") == run_id), -1)
created = runs[idx].get("createdAt") if idx >= 0 else datetime.now(UTC).isoformat()
prev_step3 = runs[idx].get("step3") if idx >= 0 else None
# 本次有效 step3:新报告优先,否则保留上次(running 中间态 step3=None 不抹掉已存的)。
effective_step3 = step3 if step3 is not None else prev_step3
run = {
"id": run_id,
"title": "后台研讨",
"createdAt": created,
"source": params.mode,
"selectedAgents": [{"agent_id": a.agent_id, "name": a.name} for a in params.agents],
"threadIds": (thread_ids or None),
"coordinatorName": params.coordinator_name,
"step2RoundtableDialogues": s2_dialogues,
"lastLeaderContent": last_leader or "",
"hasConsensus": status == "done",
"consensusPercentage": int(consensus),
"budgetLimit": 0,
"seatStubMessages": _seat_stub_messages(dialogues),
"orchestrationMode": params.mode,
"chain": params.chain,
"orchestrationPlan": params.orchestration_plan,
"step3": effective_step3,
"jobId": params.job_id,
"jobStatus": status,
}
runs = [run if i == idx else r for i, r in enumerate(runs)] if idx >= 0 else [*runs, run]
step2["runs"] = runs
step2["activeRunId"] = run_id
has_report = effective_step3 is not None
update_kwargs: dict[str, Any] = {
"step2": step2,
"furthest_step": 3 if (status == "done" and has_report) else 2,
}
if has_report:
update_kwargs["step3"] = effective_step3
# 每次尝试都带 expected_version(乐观锁);绝不降级为无版本检查的盲写。
try:
await draft_store.update_draft(
params.draft_id, params.user_id,
expected_version=expected_version,
**update_kwargs,
)
return # 写入成功
except DraftConcurrentWriteError:
if attempt < _DRAFT_WRITE_RETRIES - 1:
logger.debug(
"roundtable draft %s version conflict (attempt %d/%d), retrying",
params.draft_id, attempt + 1, _DRAFT_WRITE_RETRIES,
)
continue
# 重试全部耗尽:不盲写,置 pending 标记交由 outbox 补偿器投影。
logger.warning(
"roundtable draft %s: version conflicts exhausted after %d attempts, "
"marking job %s draft_persist_pending",
params.draft_id, _DRAFT_WRITE_RETRIES, params.job_id,
)
with suppress(Exception):
await self._store.mark_draft_persist_pending(params.job_id)
return
except Exception as exc: # noqa: BLE001 — 写草稿失败不影响作业本身
logger.warning("write run to draft %s failed", params.draft_id, exc_info=True)
from deerflow.persistence.roundtable_diagnostics import record_diagnostic
await record_diagnostic(
scope="background", stage="job", level="error", event="draft_write_failed",
message=f"研讨结果写回草稿失败(status={status}):{exc}",
detail={"type": type(exc).__name__, "error": str(exc)},
job_id=params.job_id, draft_id=params.draft_id, task_id=params.task_id, user_id=params.user_id,
)
return
def cancel(self, job_id: str) -> bool:
"""请求取消一个在跑的作业。返回是否命中在跑任务。"""
task = self._tasks.get(job_id)
if task is None or task.done():
return False
task.cancel()
return True
def is_running(self, job_id: str) -> bool:
task = self._tasks.get(job_id)
return task is not None and not task.done()
async def await_job(self, job_id: str) -> None:
"""等待某作业任务结束(测试用;生产不调用)。"""
task = self._tasks.get(job_id)
if task is not None:
try:
await task
except (Exception, asyncio.CancelledError): # noqa: BLE001 — 取消是 BaseException
pass