feat: 岗位会商命令幂等与圆桌后台作业租约调度
岗位会商写操作引入 command_id 幂等与版本锁,前端支持同键重试;圆桌后台作业改为 DB 租约 dispatcher、去重与草稿 outbox 补偿,并持久化 intent turns。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
4f536d9b39
commit
75963c9ad9
51 files changed
+4449
-670
No files matched your search
+25
@@ -0,0 +1,25 @@
|
||||
"""Intent-turn single-flight persistence (Phase 4).
|
||||
|
||||
Exposes a store factory; ``app/gateway/deps.py`` registers it on
|
||||
``app.state.intent_turn_store`` at startup. The intent router reads it from
|
||||
``request.app.state`` (no cross-boundary default needed — intent runs only in the
|
||||
gateway process).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.intent_turns.sql import IntentTurnRepository
|
||||
|
||||
|
||||
def make_intent_turn_store(session_factory) -> IntentTurnRepository | None:
|
||||
if session_factory is None:
|
||||
return None
|
||||
return IntentTurnRepository(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IntentTurnRow",
|
||||
"IntentTurnRepository",
|
||||
"make_intent_turn_store",
|
||||
]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
"""ORM model for intent-determination turn single-flight (Phase 4).
|
||||
|
||||
一条 ``intent_turns`` 行 = 第一步「确定意图」对话的**一个回合**对 ``(thread_id,
|
||||
client_turn_id)`` 的占用记录。前端为每个回合生成稳定的 ``client_turn_id``,重试 / 双击 /
|
||||
网络层重发都复用它;后端用 ``UNIQUE(thread_id, client_turn_id)`` 做跨 worker 单飞:
|
||||
|
||||
* 同一 ``(thread_id, client_turn_id)`` 至多一条行 —— INSERT 冲突即幂等命中:
|
||||
- 已 ``done`` → 原样回放缓存的终态帧,**不再启 run**(杜绝双发意图 / 重复澄清卡);
|
||||
- 在跑且租约未过期 → 该请求是重复的 in-flight 请求,丢弃(原请求仍在流式吐结果);
|
||||
- 在跑但租约过期(worker 崩溃)→ 抢占续跑;
|
||||
- ``message_hash`` 不同(同一 client_turn_id 却是不同内容)→ 409 客户端 bug。
|
||||
|
||||
一致性只来自 DB 唯一键 + 条件 UPDATE(租约 CAS),不依赖进程内 ``RunManager``
|
||||
(后者是每进程内存态,多 worker 下看不到彼此的 run,无法单飞)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class IntentTurnRow(Base):
|
||||
__tablename__ = "intent_turns"
|
||||
__table_args__ = (
|
||||
# 跨 worker 单飞的根:同一 (thread_id, client_turn_id) 至多一条。INSERT 冲突即幂等。
|
||||
UniqueConstraint("thread_id", "client_turn_id", name="uq_intent_turns_thread_client"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
# 前端每回合生成的稳定幂等键;重试 / 双击复用同一个。
|
||||
client_turn_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# 本回合 message 的 sha256:检测「同 client_turn_id 但内容变了」的客户端 bug(→ 409)。
|
||||
message_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# running(在跑)/ done(已出终态帧,可回放)/ error。
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", server_default="running"
|
||||
)
|
||||
# 终态帧的完整 JSON(done/error 时写入):重复请求命中时原样回放,不再启 run。
|
||||
result_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# 租约:领走的 worker 标记;过期(worker 崩溃)后可被条件 UPDATE 抢占续跑。
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""SQLAlchemy-backed intent-turn single-flight storage (Phase 4).
|
||||
|
||||
跨 worker 单飞 + 幂等回放,一致性只来自 DB(唯一键 + 条件 UPDATE),不依赖进程内
|
||||
``RunManager``(多 worker 下彼此不可见)。两条核心路径:
|
||||
|
||||
* ``try_claim`` —— INSERT 占用 ``(thread_id, client_turn_id)``;唯一键冲突即幂等命中,
|
||||
按「已 done→回放 / 在跑且租约有效→丢弃 / 租约过期或 error→抢占续跑 / 内容不一致→409」分流。
|
||||
* ``record_result`` —— 持租者把终态帧写回(条件 UPDATE 要求租约匹配),清租约。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
|
||||
|
||||
def _loads(value: Any) -> Any:
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
class IntentTurnRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: IntentTurnRow | None) -> dict[str, Any] | None:
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"id": row.id,
|
||||
"thread_id": row.thread_id,
|
||||
"client_turn_id": row.client_turn_id,
|
||||
"message_hash": row.message_hash,
|
||||
"status": row.status,
|
||||
"result_json": _loads(row.result_json),
|
||||
"error": row.error,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": row.lease_until.isoformat() if row.lease_until else None,
|
||||
"created_at": row.created_at,
|
||||
"updated_at": row.updated_at,
|
||||
}
|
||||
|
||||
async def get(self, thread_id: str, client_turn_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(IntentTurnRow).where(
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def try_claim(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
message_hash: str,
|
||||
lease_owner: str,
|
||||
lease_until: datetime,
|
||||
) -> dict[str, Any]:
|
||||
"""占用一个意图回合 —— 跨 worker 单飞 + 幂等的入口。
|
||||
|
||||
返回 ``{"action": ..., "row": {...}}``,``action`` 取值:
|
||||
|
||||
* ``"run"`` —— 本调用方赢得占用(新建 / 抢占续跑),应启 run 并在终态写回。
|
||||
* ``"replay"`` —— 该回合已 ``done``,原样回放 ``row.result_json``,**不再启 run**。
|
||||
* ``"duplicate"``—— 该回合仍在跑且租约有效(或刚被别处抢占),本请求是重复的
|
||||
in-flight 请求,应丢弃(原请求仍在流式吐结果)。
|
||||
* ``"conflict"`` —— 同一 ``client_turn_id`` 但 ``message_hash`` 不同 → 客户端 bug,409。
|
||||
"""
|
||||
# 1) 先尝试 INSERT 占用。
|
||||
new_id = uuid4().hex
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(
|
||||
IntentTurnRow(
|
||||
id=new_id,
|
||||
thread_id=thread_id,
|
||||
client_turn_id=client_turn_id,
|
||||
message_hash=message_hash,
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
row = await self.get(thread_id, client_turn_id)
|
||||
return {"action": "run", "row": row}
|
||||
except IntegrityError:
|
||||
# 唯一键 (thread_id, client_turn_id) 冲突 → 幂等命中,读回既有行分流。
|
||||
pass
|
||||
|
||||
existing = await self.get(thread_id, client_turn_id)
|
||||
if existing is None:
|
||||
# 极端竞态:冲突与回读之间行被清掉。降级为「可新建」再插一次(best-effort)。
|
||||
return {"action": "run", "row": None}
|
||||
|
||||
# 2) 内容指纹不一致 → 客户端 bug(同 client_turn_id 却换了 message)。
|
||||
if existing.get("message_hash") != message_hash:
|
||||
return {"action": "conflict", "row": existing}
|
||||
|
||||
status = existing.get("status") or "running"
|
||||
|
||||
# 3) 已完成 → 幂等回放终态帧。
|
||||
if status == "done":
|
||||
return {"action": "replay", "row": existing}
|
||||
|
||||
# 4) 在跑或出错 → 尝试条件 UPDATE 抢占续跑(error 强制;running 要求租约已过期)。
|
||||
reclaimed = await self._conditional_reclaim(
|
||||
thread_id=thread_id,
|
||||
client_turn_id=client_turn_id,
|
||||
prev_status=status,
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
)
|
||||
if reclaimed is not None:
|
||||
return {"action": "run", "row": reclaimed}
|
||||
|
||||
# 5) 抢占失败:仍在跑且租约有效(或刚被别处抢占)→ 本请求是重复 in-flight,丢弃。
|
||||
return {"action": "duplicate", "row": existing}
|
||||
|
||||
async def _conditional_reclaim(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
prev_status: str,
|
||||
lease_owner: str,
|
||||
lease_until: datetime,
|
||||
) -> dict[str, Any] | None:
|
||||
"""原子条件 UPDATE 抢占:``error`` 强制续跑;``running`` 仅当租约已过期。
|
||||
|
||||
成功则置 ``status='running'``、写新租约,返回行;否则 None(仍在跑 / 刚被抢占)。
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
where = [
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
]
|
||||
if prev_status == "error":
|
||||
where.append(IntentTurnRow.status == "error")
|
||||
else:
|
||||
where.extend(
|
||||
[
|
||||
IntentTurnRow.status == "running",
|
||||
IntentTurnRow.lease_until.is_not(None),
|
||||
IntentTurnRow.lease_until < now,
|
||||
]
|
||||
)
|
||||
values = {
|
||||
"status": "running",
|
||||
"lease_owner": lease_owner,
|
||||
"lease_until": lease_until,
|
||||
"updated_at": now,
|
||||
}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(IntentTurnRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return None
|
||||
await session.commit()
|
||||
row = (
|
||||
await session.execute(
|
||||
select(IntentTurnRow).where(
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def record_result(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
client_turn_id: str,
|
||||
lease_owner: str,
|
||||
status: str,
|
||||
result_json: Any = None,
|
||||
error: str | None = None,
|
||||
) -> bool:
|
||||
"""持租者把终态帧写回(条件 UPDATE 要求租约匹配),并清租约。
|
||||
|
||||
只有赢得 ``try_claim``(``action='run'``)的调用方才应调用,且 ``lease_owner``
|
||||
必须与领取时一致 —— 防止「抢占续跑的新 worker」被「旧 worker 的迟到收尾」覆盖。
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {
|
||||
"status": status,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
if result_json is not None:
|
||||
values["result_json"] = json.dumps(result_json, ensure_ascii=False)
|
||||
if error is not None:
|
||||
values["error"] = error
|
||||
where = [
|
||||
IntentTurnRow.thread_id == thread_id,
|
||||
IntentTurnRow.client_turn_id == client_turn_id,
|
||||
IntentTurnRow.lease_owner == lease_owner,
|
||||
IntentTurnRow.status == "running",
|
||||
]
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(IntentTurnRow).where(*where).values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
Reference in new issue
Block a user