feat: 岗位会商命令幂等与圆桌后台作业租约调度
岗位会商写操作引入 command_id 幂等与版本锁,前端支持同键重试;圆桌后台作业改为 DB 租约 dispatcher、去重与草稿 outbox 补偿,并持久化 intent turns。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
4f536d9b39
commit
75963c9ad9
51 files changed
+4449
-670
No files matched your search
@@ -18,6 +18,7 @@ Routes(prefix ``/api/roundtable-jobs``)::
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
@@ -25,11 +26,11 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, HTTPException, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.roundtable_job_executor import StartParams
|
||||
from app.gateway.roundtable_job_executor import build_start_snapshot
|
||||
from deerflow.agents.roundtable_orchestrator import SeatRef
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
@@ -54,6 +55,10 @@ class JobAgent(BaseModel):
|
||||
|
||||
|
||||
class StartJobRequest(BaseModel):
|
||||
# Phase 3 客户端幂等键:前端为每次「后台挂起」生成一个 requestId,网络重试**复用同一
|
||||
# requestId**。服务端据此审计 / 排查;「同草稿不双跑」本身由 active_dedupe_key 唯一索引
|
||||
# 保证(requestId 只是锦上添花的请求级指纹)。缺省(旧前端)→ 服务端生成一个。
|
||||
requestId: str | None = Field(default=None, max_length=64)
|
||||
draftId: str | None = Field(default=None, max_length=64)
|
||||
# 外部任务 id(taskId 深链会商)。非空 = task 作业:按 task_id 共享、不按 user 分权,
|
||||
# 其 draft_id 指向独立的 roundtable_task_drafts 表(最终报告写回那里)。
|
||||
@@ -151,6 +156,20 @@ def _get_executor(request: Request):
|
||||
return executor
|
||||
|
||||
|
||||
def _get_dispatcher(request: Request):
|
||||
"""Phase 3 dispatcher(可能为 None:store 未配置时)。nudge 前先判空。"""
|
||||
return getattr(request.app.state, "roundtable_job_dispatcher", None)
|
||||
|
||||
|
||||
def _request_hash(body: StartJobRequest) -> str:
|
||||
"""启动入参指纹(sha256)——审计/排查「同 requestId 但入参变了」的异常重试。"""
|
||||
canonical = json.dumps(
|
||||
body.model_dump(mode="json", exclude={"requestId"}),
|
||||
ensure_ascii=False, sort_keys=True, default=str,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _pending_nodes(agents: list[JobAgent]) -> list[dict[str, Any]]:
|
||||
"""启动即落一份「全 pending」的派活链路,使 GET / 首帧 SSE 立刻能渲染流程图。"""
|
||||
return [
|
||||
@@ -194,32 +213,63 @@ def _seat_refs_from_nodes(nodes: list[dict[str, Any]] | None) -> list[SeatRef]:
|
||||
|
||||
|
||||
@router.post("", response_model=RoundtableJobResponse, status_code=201)
|
||||
async def start_job(request: Request, body: StartJobRequest) -> RoundtableJobResponse:
|
||||
async def start_job(
|
||||
request: Request, body: StartJobRequest, response: Response
|
||||
) -> RoundtableJobResponse:
|
||||
"""启动一个后台作业(Phase 3:唯一活跃键去重 + dispatcher 领取执行)。
|
||||
|
||||
流程:
|
||||
1. 把整份启动入参序列化成 ``input_snapshot`` 落库(跨 worker 重建 StartParams 的前提);
|
||||
2. ``try_create_or_get_active`` 直接 INSERT 并占用 ``active_dedupe_key`` 唯一索引 ——
|
||||
冲突(同草稿已有活跃作业)即幂等返回它(``created=False`` → HTTP 200),不再新建;
|
||||
3. 新建成功(HTTP 201,status=queued)后 ``nudge`` dispatcher,由它 ``claim_job``
|
||||
原子领取并在本(或其它)worker 执行 —— **不再由路由直接拉起进程内任务**。
|
||||
|
||||
「同一草稿至多一个活跃作业」由 DB 唯一索引强制,跨 worker / 并发 / 重试天然互斥,
|
||||
不依赖 GET_LOCK 命名锁或进程内 asyncio。
|
||||
"""
|
||||
store = _get_store(request)
|
||||
executor = _get_executor(request)
|
||||
user_id = _current_user_id(request)
|
||||
|
||||
if not body.agents:
|
||||
raise HTTPException(status_code=400, detail="agents is required")
|
||||
|
||||
# 防重复挂起(服务端幂等闸门):同一草稿已有未终态作业(queued/running/awaiting_input)
|
||||
# 时直接返回它、不再新建。前端的「手动挂起按钮 / 离开页面自动挂起 / beforeunload 刷新
|
||||
# 兜底」可能对同一研讨多次触发本端点 —— 客户端去重只能挡住同页内的重复,跨触发源 /
|
||||
# 跨标签页的并发只能靠这里兜住,否则同一研讨会被双跑。
|
||||
task_id = (body.taskId or "").strip() or None
|
||||
if body.draftId:
|
||||
# task 作业按 task_id 去重(跨用户共享);个人作业按 user 去重。
|
||||
existing = await store.get_by_draft(body.draftId, user_id=user_id, task_id=task_id)
|
||||
if existing is not None and (existing.get("status") or "") not in _TERMINAL:
|
||||
return _to_response(existing)
|
||||
|
||||
draft_id = (body.draftId or "").strip() or None
|
||||
agents = [SeatRef(a.agentId, a.name or a.agentId) for a in body.agents]
|
||||
seed_message = (body.seedMessage or body.intent or "").strip()
|
||||
|
||||
# 整份启动入参快照 —— dispatcher 在任意 worker 领取后据此无损重建 StartParams。
|
||||
snapshot = build_start_snapshot(
|
||||
user_id=user_id,
|
||||
agents=agents,
|
||||
seed_message=seed_message,
|
||||
model=body.model,
|
||||
mode=body.orchestrationMode,
|
||||
intent_text=body.intent,
|
||||
thread_ids=body.threadIds,
|
||||
coordinator_name=body.coordinatorName,
|
||||
draft_id=draft_id,
|
||||
task_id=task_id,
|
||||
is_resume=False,
|
||||
chain=body.chain,
|
||||
orchestration_plan=body.orchestrationPlan,
|
||||
seat_thinking_enabled=body.thinking_enabled,
|
||||
seat_reasoning_effort=body.reasoning_effort,
|
||||
seat_subagent_enabled=body.subagent_enabled,
|
||||
seat_skill_directives=dict(body.seat_skill_directives or {}),
|
||||
seat_modes=dict(body.seat_modes or {}),
|
||||
gather_first=bool(body.gather_first),
|
||||
business_code=(body.business_code or None),
|
||||
)
|
||||
request_id = (body.requestId or "").strip() or uuid4().hex
|
||||
|
||||
job_id = uuid4().hex
|
||||
row = await store.create(
|
||||
id=job_id,
|
||||
draft_id=body.draftId,
|
||||
row, created = await store.try_create_or_get_active(
|
||||
draft_id=draft_id,
|
||||
user_id=user_id,
|
||||
task_id=task_id,
|
||||
id=job_id,
|
||||
orchestration_mode=body.orchestrationMode,
|
||||
coordinator_name=body.coordinatorName,
|
||||
thread_ids=body.threadIds,
|
||||
@@ -227,32 +277,21 @@ async def start_job(request: Request, body: StartJobRequest) -> RoundtableJobRes
|
||||
orchestration_plan=body.orchestrationPlan,
|
||||
dispatch_chain=_pending_nodes(body.agents),
|
||||
status="queued",
|
||||
request_id=request_id,
|
||||
request_hash=_request_hash(body),
|
||||
input_snapshot=snapshot,
|
||||
)
|
||||
|
||||
executor.start_job(
|
||||
StartParams(
|
||||
job_id=job_id,
|
||||
user_id=user_id,
|
||||
agents=agents,
|
||||
seed_message=(body.seedMessage or body.intent or "").strip(),
|
||||
model=body.model,
|
||||
mode=body.orchestrationMode,
|
||||
intent_text=body.intent,
|
||||
thread_ids=body.threadIds,
|
||||
coordinator_name=body.coordinatorName,
|
||||
draft_id=body.draftId,
|
||||
task_id=task_id,
|
||||
chain=body.chain,
|
||||
orchestration_plan=body.orchestrationPlan,
|
||||
seat_thinking_enabled=body.thinking_enabled,
|
||||
seat_reasoning_effort=body.reasoning_effort,
|
||||
seat_subagent_enabled=body.subagent_enabled,
|
||||
seat_skill_directives=dict(body.seat_skill_directives or {}),
|
||||
seat_modes=dict(body.seat_modes or {}),
|
||||
gather_first=bool(body.gather_first),
|
||||
business_code=(body.business_code or None),
|
||||
)
|
||||
)
|
||||
if created:
|
||||
# 新建(queued):交给 dispatcher 领取执行。
|
||||
response.status_code = 201
|
||||
else:
|
||||
# 命中已有活跃作业:幂等返回(不重复建 / 不重复跑)。
|
||||
response.status_code = 200
|
||||
# 任一分支都唤醒 dispatcher:新建需被领取;幂等命中时若既有作业还在 queued、未被任何
|
||||
# worker 领走(如 dispatcher 正在轮询间隙),nudge 确保它被立刻 claim,无感延迟。
|
||||
dispatcher = _get_dispatcher(request)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
return _to_response(row)
|
||||
|
||||
|
||||
@@ -310,48 +349,73 @@ async def stream_job(request: Request, job_id: str, task_id: str | None = None)
|
||||
async def resume_job(
|
||||
request: Request, job_id: str, body: ResumeJobRequest, task_id: str | None = None
|
||||
) -> RoundtableJobResponse:
|
||||
"""回答澄清后续跑(Phase 3:条件 UPDATE 串行 + dispatcher 领取)。
|
||||
|
||||
``queue_for_resume`` 用**原子条件 UPDATE**(WHERE status='awaiting_input')把作业翻成
|
||||
queued —— 并发 / 重复的 resume 只有一个能成功(其余返回 None → 409),杜绝「同一澄清
|
||||
被续跑两次」的双跑。成功后清租约 + 写含续跑语义的新 snapshot,``nudge`` dispatcher 由
|
||||
它在任意 worker 领取执行;**路由不再直接拉起进程内任务**。
|
||||
"""
|
||||
store = _get_store(request)
|
||||
executor = _get_executor(request)
|
||||
tid = (task_id or "").strip() or None
|
||||
user_id = _current_user_id(request)
|
||||
|
||||
row = await store.get(job_id, user_id=user_id, task_id=tid)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
if (row.get("status") or "") != "awaiting_input":
|
||||
raise HTTPException(status_code=409, detail="Job is not awaiting input")
|
||||
|
||||
answer = body.answer.strip()
|
||||
if not answer:
|
||||
raise HTTPException(status_code=400, detail="answer is required")
|
||||
|
||||
# task 作业不分权:放开写入归属校验(user_id=None);并以**原作业所有者**身份续跑,
|
||||
# 保证编排上下文 / 写回 task-draft 用一致的 user 上下文。
|
||||
# task 作业不分权:放开写入归属校验(user_id=None)。
|
||||
write_user_id = None if tid else user_id
|
||||
run_user_id = row.get("user_id") or user_id
|
||||
updated = await store.update_progress(
|
||||
job_id, user_id=write_user_id, status="running", pending_clarification=None
|
||||
)
|
||||
|
||||
executor.start_job(
|
||||
StartParams(
|
||||
job_id=job_id,
|
||||
user_id=run_user_id,
|
||||
agents=_seat_refs_from_nodes(row.get("dispatch_chain")),
|
||||
seed_message=_RESUME_PREFIX + answer,
|
||||
model=None,
|
||||
mode=row.get("orchestration_mode") or "recommend",
|
||||
intent_text=answer,
|
||||
thread_ids=row.get("thread_ids"),
|
||||
coordinator_name=row.get("coordinator_name") or "roundtable-coordinator",
|
||||
draft_id=row.get("draft_id"),
|
||||
task_id=tid or row.get("task_id"),
|
||||
is_resume=True,
|
||||
chain=row.get("chain"),
|
||||
orchestration_plan=row.get("orchestration_plan"),
|
||||
)
|
||||
# 续跑 snapshot:优先复用原 input_snapshot(含席位/模式/链条等完整入参),覆盖续跑三件套
|
||||
# (is_resume / seed_message / intent_text),并用作业行里**最新**的 thread_ids /
|
||||
# coordinator_name(首跑建线程后由执行器写回,比原 snapshot 里的更新)。旧作业无 snapshot
|
||||
# 时从作业行字段兜底重建。
|
||||
base_snapshot = row.get("input_snapshot") or {}
|
||||
if not isinstance(base_snapshot, dict):
|
||||
base_snapshot = {}
|
||||
resume_snapshot: dict[str, Any] = {
|
||||
"v": base_snapshot.get("v", 1),
|
||||
"user_id": run_user_id,
|
||||
"agents": base_snapshot.get("agents")
|
||||
or [{"agent_id": n.get("agentId"), "name": n.get("name")} for n in (row.get("dispatch_chain") or [])],
|
||||
"seed_message": _RESUME_PREFIX + answer,
|
||||
"model": base_snapshot.get("model"),
|
||||
"mode": row.get("orchestration_mode") or base_snapshot.get("mode") or "recommend",
|
||||
"intent_text": answer,
|
||||
"thread_ids": row.get("thread_ids"),
|
||||
"coordinator_name": row.get("coordinator_name") or "roundtable-coordinator",
|
||||
"draft_id": row.get("draft_id"),
|
||||
"task_id": tid or row.get("task_id"),
|
||||
"is_resume": True,
|
||||
"chain": row.get("chain"),
|
||||
"orchestration_plan": row.get("orchestration_plan"),
|
||||
"enable_report": base_snapshot.get("enable_report", True),
|
||||
"seat_thinking_enabled": base_snapshot.get("seat_thinking_enabled"),
|
||||
"seat_reasoning_effort": base_snapshot.get("seat_reasoning_effort"),
|
||||
"seat_subagent_enabled": base_snapshot.get("seat_subagent_enabled", False),
|
||||
"seat_skill_directives": base_snapshot.get("seat_skill_directives") or {},
|
||||
"seat_modes": base_snapshot.get("seat_modes") or {},
|
||||
"gather_first": base_snapshot.get("gather_first", False),
|
||||
"business_code": base_snapshot.get("business_code"),
|
||||
}
|
||||
|
||||
updated = await store.queue_for_resume(
|
||||
job_id, input_snapshot=resume_snapshot, user_id=write_user_id, task_id=tid
|
||||
)
|
||||
return _to_response(updated or row)
|
||||
if updated is None:
|
||||
# 条件 UPDATE 未命中:作业已不在 awaiting_input(被并发 resume 抢先 / 已续跑 / 已终态)。
|
||||
raise HTTPException(status_code=409, detail="Job is not awaiting input")
|
||||
|
||||
dispatcher = _get_dispatcher(request)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
return _to_response(updated)
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
@@ -395,8 +459,19 @@ async def admin_cleanup(request: Request, body: CleanupRequest) -> CleanupRespon
|
||||
|
||||
@router.post("/{job_id}/cancel", response_model=RoundtableJobResponse)
|
||||
async def cancel_job(request: Request, job_id: str, task_id: str | None = None) -> RoundtableJobResponse:
|
||||
"""取消作业(Phase 3:两段式 cancel)。
|
||||
|
||||
1. ``request_cancel``:原子条件 UPDATE(可取消态 → ``cancel_requested``)—— 权威意图落库,
|
||||
重复 cancel 幂等(已 cancel_requested/cancelled/终态时返回 None,直接回现状)。
|
||||
2. **无持租者**(queued / awaiting_input,执行器未在跑)→ 路由立刻 ``finalize_cancel``
|
||||
收口为终态 ``cancelled``(清唯一活跃键 + 租约)。
|
||||
3. **有持租者**(running)→ 保留 ``cancel_requested``,由持租者的 DB 看门狗收口
|
||||
``cancelled``;本接口先回 ``cancel_requested``(SSE 随后见翻转)。租约过期前未能
|
||||
收口(例如 worker 崩溃)时,dispatcher 会回收该取消请求。
|
||||
|
||||
「已取消」终态永远由持租者或 finalize 收口,避免与执行器的进度写竞态。
|
||||
"""
|
||||
store = _get_store(request)
|
||||
executor = _get_executor(request)
|
||||
tid = (task_id or "").strip() or None
|
||||
user_id = _current_user_id(request)
|
||||
|
||||
@@ -404,7 +479,20 @@ async def cancel_job(request: Request, job_id: str, task_id: str | None = None)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
executor.cancel(job_id)
|
||||
# task 作业不分权:放开写入归属校验。
|
||||
updated = await store.update_progress(job_id, user_id=(None if tid else user_id), status="cancelled")
|
||||
return _to_response(updated or row)
|
||||
write_user_id = None if tid else user_id
|
||||
requested = await store.request_cancel(job_id, user_id=write_user_id, task_id=tid)
|
||||
if requested is None:
|
||||
# 已不可取消(终态 / 已 cancel_requested):幂等返回现状,不当作错误。
|
||||
fresh = await store.get(job_id, user_id=user_id, task_id=tid)
|
||||
return _to_response(fresh or row)
|
||||
|
||||
if not requested.get("lease_owner"):
|
||||
# 无持租者:路由直接收口(queued / awaiting_input)。
|
||||
finalized = await store.finalize_cancel(job_id)
|
||||
return _to_response(finalized or requested)
|
||||
|
||||
# 有持租者(running):不要直接 task.cancel()。直接取消会中断当前 worker 的看门狗,
|
||||
# 使 DB 永久遗留在 cancel_requested;让 DB 看门狗以持租者身份收口,崩溃时由 dispatcher
|
||||
# 在租约到期后回收,才能保证跨进程取消最终完成。
|
||||
return _to_response(requested)
|
||||
Reference in new issue
Block a user