feat(maf): complete resilient roundtable workflow
This commit is contained in:
1 parent
bd4131b4e2
commit
21c4c898e7
23 files changed
+4810
-312
No files matched your search
+12
@@ -62,6 +62,18 @@ class MafRoundtableConfig(BaseModel):
|
||||
default=True,
|
||||
description="管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)",
|
||||
)
|
||||
# Phase 3: per-seat wall-clock timeout (seconds) for structured expert turns.
|
||||
seat_timeout_seconds: int = Field(
|
||||
default=600,
|
||||
ge=30,
|
||||
le=7200,
|
||||
description="结构化流程中单个专家席位的超时秒数;超时记为 partial_failure 并继续其余席位",
|
||||
)
|
||||
# Phase 3: allow one directed format-repair turn when seat JSON fails schema.
|
||||
schema_repair_enabled: bool = Field(
|
||||
default=True,
|
||||
description="席位输出未通过 SeatTurnOutput 校验时是否允许一次定向格式修复",
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+37
-3
@@ -30,6 +30,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
@@ -306,13 +307,34 @@ class MafRoundtableSeatRunRow(Base):
|
||||
the system can still locate and cancel outstanding DeerFlow runs, and the
|
||||
MAF checkpoint can reference which seat turns completed.
|
||||
|
||||
Unique constraint: ``(maf_run_id, agent_id, turn_index)`` — one DeerFlow run
|
||||
per seat per turn.
|
||||
Phase-3 unique constraint: ``(maf_run_id, agent_id, turn_index,
|
||||
execution_stage)``. ``turn_index`` is the business round (kept stable for
|
||||
frontend display / approval matching / event tracing); ``execution_stage``
|
||||
disambiguates the distinct DeerFlow executions that happen within one
|
||||
business round (planner / expert / format_repair / cross_review /
|
||||
finalizer / group_chat). The business turn_index is NEVER mutated to dodge
|
||||
a collision.
|
||||
"""
|
||||
|
||||
__tablename__ = "maf_roundtable_seat_runs"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("maf_run_id", "agent_id", "turn_index", name="uq_maf_seat_runs_run_agent_turn"),
|
||||
UniqueConstraint(
|
||||
"maf_run_id",
|
||||
"agent_id",
|
||||
"turn_index",
|
||||
"execution_stage",
|
||||
name="uq_maf_seat_runs_run_agent_turn_stage",
|
||||
),
|
||||
# Phase-3 P1-2: enforce the closed execution_stage enum at the DB layer.
|
||||
# Mirrors seat_runs.VALID_EXECUTION_STAGES — a typo (e.g. "exprt") would
|
||||
# otherwise silently bypass the four-column idempotency key by creating
|
||||
# a brand-new stage. This CHECK keeps both fresh creates and any raw
|
||||
# migration / manual SQL honest. Keep in sync with VALID_EXECUTION_STAGES.
|
||||
CheckConstraint(
|
||||
"execution_stage IN ('legacy','planner','expert','format_repair',"
|
||||
"'cross_review','cross_review_repair','finalizer','group_chat')",
|
||||
name="ck_maf_seat_runs_execution_stage",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
@@ -321,12 +343,24 @@ class MafRoundtableSeatRunRow(Base):
|
||||
)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
turn_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
# Phase-3: which kind of DeerFlow execution this row represents. Pre-Phase-3
|
||||
# rows are migrated to ``legacy``. Closed enum (see seat_runs.VALID_EXECUTION_STAGES):
|
||||
# planner | expert | format_repair | cross_review | cross_review_repair |
|
||||
# finalizer | group_chat | legacy.
|
||||
execution_stage: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="legacy", server_default="legacy"
|
||||
)
|
||||
deerflow_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
deerflow_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
sandbox_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
workflow_checkpoint_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# pending | running | completed | failed | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
# Phase-3 P0-1: a sanitized, replayable snapshot of the SeatResponse written
|
||||
# atomically with the completed transition. Survives a crash that happens
|
||||
# after the seat run is marked completed but before the workflow checkpoint
|
||||
# is persisted, so recovery never needs to re-call DeerFlow.
|
||||
result_snapshot: Mapped[dict | None] = mapped_column(PortableJSON, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
+249
-8
@@ -12,6 +12,7 @@ from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.maf_roundtable.model import MafRoundtableSeatRunRow
|
||||
@@ -21,17 +22,68 @@ def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
# ── execution_stage enum (Phase-3 P1-2) ─────────────────────────────────────
|
||||
# The fixed set of distinct DeerFlow executions that may occur within one
|
||||
# business round (turn_index). The four-column idempotency key
|
||||
# (maf_run_id, agent_id, turn_index, execution_stage) only works if stages
|
||||
# are a closed set — a typo would silently create a new stage and bypass the
|
||||
# dedup. Validate on write so a misspelling raises loudly instead.
|
||||
LEGACY = "legacy"
|
||||
PLANNER = "planner"
|
||||
EXPERT = "expert"
|
||||
FORMAT_REPAIR = "format_repair"
|
||||
CROSS_REVIEW = "cross_review"
|
||||
CROSS_REVIEW_REPAIR = "cross_review_repair"
|
||||
FINALIZER = "finalizer"
|
||||
GROUP_CHAT = "group_chat"
|
||||
|
||||
VALID_EXECUTION_STAGES: frozenset[str] = frozenset({
|
||||
LEGACY,
|
||||
PLANNER,
|
||||
EXPERT,
|
||||
FORMAT_REPAIR,
|
||||
CROSS_REVIEW,
|
||||
CROSS_REVIEW_REPAIR,
|
||||
FINALIZER,
|
||||
GROUP_CHAT,
|
||||
})
|
||||
|
||||
|
||||
class InvalidExecutionStageError(ValueError):
|
||||
"""Raised when an execution_stage outside the closed enum is written."""
|
||||
|
||||
|
||||
class SeatRunNotFoundError(LookupError):
|
||||
"""Raised when a seat run expected by the four-tuple does not exist."""
|
||||
|
||||
|
||||
class SeatRunWaitTimeout(TimeoutError):
|
||||
"""Raised when waiting for a seat run to reach a terminal state times out."""
|
||||
|
||||
|
||||
def _validate_stage(execution_stage: str) -> str:
|
||||
if execution_stage not in VALID_EXECUTION_STAGES:
|
||||
raise InvalidExecutionStageError(
|
||||
f"execution_stage={execution_stage!r} is not one of the allowed "
|
||||
f"stages {sorted(VALID_EXECUTION_STAGES)}; a typo here would bypass "
|
||||
f"the four-column idempotency key."
|
||||
)
|
||||
return execution_stage
|
||||
|
||||
|
||||
def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"maf_run_id": row.maf_run_id,
|
||||
"agent_id": row.agent_id,
|
||||
"turn_index": row.turn_index,
|
||||
"execution_stage": row.execution_stage,
|
||||
"deerflow_run_id": row.deerflow_run_id,
|
||||
"deerflow_thread_id": row.deerflow_thread_id,
|
||||
"sandbox_id": row.sandbox_id,
|
||||
"workflow_checkpoint_id": row.workflow_checkpoint_id,
|
||||
"status": row.status,
|
||||
"result_snapshot": row.result_snapshot,
|
||||
"started_at": row.started_at.isoformat() if isinstance(row.started_at, datetime) else row.started_at,
|
||||
"finished_at": row.finished_at.isoformat() if isinstance(row.finished_at, datetime) else row.finished_at,
|
||||
"error": row.error,
|
||||
@@ -40,6 +92,12 @@ def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
# Seat-run lifecycle statuses that mean "still in flight" — a second attempt to
|
||||
# start the same DeerFlow execution must NOT launch another run while one of
|
||||
# these exists.
|
||||
_ACTIVE_STATUSES = ("pending", "running")
|
||||
|
||||
|
||||
class MafSeatRunRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -50,14 +108,17 @@ class MafSeatRunRepository:
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str = "legacy",
|
||||
deerflow_thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
_validate_stage(execution_stage)
|
||||
now = _now()
|
||||
row = MafRoundtableSeatRunRow(
|
||||
id=uuid4().hex,
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
deerflow_thread_id=deerflow_thread_id,
|
||||
status="pending",
|
||||
created_at=now,
|
||||
@@ -68,6 +129,171 @@ class MafSeatRunRepository:
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def get_by_four_tuple(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Look up a seat run by its Phase-3 identity key (no DeerFlow side effects)."""
|
||||
stmt = select(MafRoundtableSeatRunRow).where(
|
||||
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
|
||||
MafRoundtableSeatRunRow.agent_id == agent_id,
|
||||
MafRoundtableSeatRunRow.turn_index == turn_index,
|
||||
MafRoundtableSeatRunRow.execution_stage == execution_stage,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def wait_for_terminal_seat_run(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
timeout_seconds: float,
|
||||
poll_interval_seconds: float = 0.1,
|
||||
) -> dict[str, Any]:
|
||||
"""Block until the seat run identified by the four-tuple reaches a terminal status.
|
||||
|
||||
Terminal statuses: completed, failed, cancelled, cancel_failed, cancel_unknown.
|
||||
Non-terminal (still in-flight): pending, running, cancelling.
|
||||
|
||||
Raises:
|
||||
SeatRunNotFoundError: if no row exists for the four-tuple.
|
||||
SeatRunWaitTimeout: if the deadline is reached while still non-terminal.
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
_TERMINAL = ("completed", "failed", "cancelled", "cancel_failed", "cancel_unknown")
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
|
||||
while True:
|
||||
row = await self.get_by_four_tuple(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if row is None:
|
||||
raise SeatRunNotFoundError(
|
||||
f"seat run not found: maf_run_id={maf_run_id!r} agent_id={agent_id!r} "
|
||||
f"turn_index={turn_index} execution_stage={execution_stage!r}"
|
||||
)
|
||||
if row["status"] in _TERMINAL:
|
||||
return row
|
||||
if time.monotonic() >= deadline:
|
||||
raise SeatRunWaitTimeout(
|
||||
f"timed out waiting for seat run {row['id']!r} "
|
||||
f"(stage={execution_stage!r}, status={row['status']!r}) "
|
||||
f"after {timeout_seconds}s"
|
||||
)
|
||||
await asyncio.sleep(poll_interval_seconds)
|
||||
|
||||
async def get_or_create_seat_run(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
deerflow_thread_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""Atomic idempotent seat-run acquisition (Phase-3 P0-1).
|
||||
|
||||
Race-free: tries to INSERT first; on a four-column unique-key conflict
|
||||
it rolls back the short transaction and reads the existing row. This is
|
||||
strictly stronger than "query-then-insert" because two concurrent
|
||||
callers that both miss the lookup would otherwise both insert — only
|
||||
the DB constraint decides the winner, and the loser reads the winner's
|
||||
row.
|
||||
|
||||
Because the winner's transaction may not have committed by the time the
|
||||
loser's post-conflict read runs (session isolation), the loser retries
|
||||
the read a bounded number of times with a short backoff before giving
|
||||
up. ``IntegrityError`` is NEVER propagated to the caller on a four-tuple
|
||||
conflict.
|
||||
|
||||
Returns ``(seat_run, created)``:
|
||||
- ``created=True`` → caller OWNS this execution and may start DeerFlow.
|
||||
- ``created=False`` → a row already exists; the caller must inspect its
|
||||
``status`` and must NOT start a second DeerFlow run for it.
|
||||
"""
|
||||
_validate_stage(execution_stage)
|
||||
now = _now()
|
||||
row = MafRoundtableSeatRunRow(
|
||||
id=uuid4().hex,
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
deerflow_thread_id=deerflow_thread_id,
|
||||
status="pending",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Lost the race (or replayed an existing four-tuple). Roll back
|
||||
# this short transaction, then read the winner's row. The winner
|
||||
# may not have committed yet (session isolation), so retry a few
|
||||
# times with a short backoff before treating this as a hard
|
||||
# error.
|
||||
await session.rollback()
|
||||
existing = await self._await_existing(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if existing is None:
|
||||
# The conflict was NOT on our four-tuple (e.g. a FK
|
||||
# violation) — surface it as a clean error rather than
|
||||
# swallowing it.
|
||||
raise
|
||||
return existing, False
|
||||
return _to_dict(row), True
|
||||
|
||||
async def _await_existing(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
retries: int = 10,
|
||||
backoff_s: float = 0.01,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Read the winner's row, retrying briefly until it is visible.
|
||||
|
||||
Under true concurrency the winner's COMMIT may land after the loser's
|
||||
post-rollback SELECT; the loser cannot see an uncommitted row, so we
|
||||
poll with a tiny backoff until the winner's transaction commits (or the
|
||||
retry budget is exhausted, in which case we return None so the caller
|
||||
can decide it was a non-four-tuple conflict).
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
for _ in range(max(1, retries)):
|
||||
existing = await self.get_by_four_tuple(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if existing is not None:
|
||||
return existing
|
||||
await asyncio.sleep(backoff_s)
|
||||
return None
|
||||
|
||||
async def mark_running(
|
||||
self, seat_run_id: str, *, deerflow_run_id: str, sandbox_id: str | None = None
|
||||
) -> None:
|
||||
@@ -85,17 +311,29 @@ class MafSeatRunRepository:
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def mark_completed(self, seat_run_id: str, *, checkpoint_id: str | None = None) -> None:
|
||||
async def mark_completed(
|
||||
self,
|
||||
seat_run_id: str,
|
||||
*,
|
||||
checkpoint_id: str | None = None,
|
||||
result_snapshot: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Mark a seat run completed, optionally persisting a sanitized result
|
||||
snapshot so it can be replayed after a crash even if the workflow
|
||||
checkpoint was not yet written."""
|
||||
values: dict[str, Any] = {
|
||||
"status": "completed",
|
||||
"workflow_checkpoint_id": checkpoint_id,
|
||||
"finished_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
if result_snapshot is not None:
|
||||
values["result_snapshot"] = result_snapshot
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(MafRoundtableSeatRunRow)
|
||||
.where(MafRoundtableSeatRunRow.id == seat_run_id)
|
||||
.values(
|
||||
status="completed",
|
||||
workflow_checkpoint_id=checkpoint_id,
|
||||
finished_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
@@ -163,7 +401,10 @@ class MafSeatRunRepository:
|
||||
stmt = (
|
||||
select(MafRoundtableSeatRunRow)
|
||||
.where(MafRoundtableSeatRunRow.maf_run_id == maf_run_id)
|
||||
.order_by(MafRoundtableSeatRunRow.turn_index.asc())
|
||||
.order_by(
|
||||
MafRoundtableSeatRunRow.turn_index.asc(),
|
||||
MafRoundtableSeatRunRow.execution_stage.asc(),
|
||||
)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
Reference in new issue
Block a user