feat(maf): complete resilient roundtable workflow

This commit is contained in:
董宏凯 committed 2026-08-02 00:23:25 +08:00
1 parent bd4131b4e2
commit 21c4c898e7
23 files changed
+4810 -312

No files matched your search

@@ -62,6 +62,18 @@ class MafRoundtableConfig(BaseModel):
default=True,
description="管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)",
)
# Phase 3: per-seat wall-clock timeout (seconds) for structured expert turns.
seat_timeout_seconds: int = Field(
default=600,
ge=30,
le=7200,
description="结构化流程中单个专家席位的超时秒数;超时记为 partial_failure 并继续其余席位",
)
# Phase 3: allow one directed format-repair turn when seat JSON fails schema.
schema_repair_enabled: bool = Field(
default=True,
description="席位输出未通过 SeatTurnOutput 校验时是否允许一次定向格式修复",
)
__all__ = [
@@ -30,6 +30,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import (
CheckConstraint,
ForeignKey,
Integer,
String,
@@ -306,13 +307,34 @@ class MafRoundtableSeatRunRow(Base):
the system can still locate and cancel outstanding DeerFlow runs, and the
MAF checkpoint can reference which seat turns completed.
Unique constraint: ``(maf_run_id, agent_id, turn_index)`` — one DeerFlow run
per seat per turn.
Phase-3 unique constraint: ``(maf_run_id, agent_id, turn_index,
execution_stage)``. ``turn_index`` is the business round (kept stable for
frontend display / approval matching / event tracing); ``execution_stage``
disambiguates the distinct DeerFlow executions that happen within one
business round (planner / expert / format_repair / cross_review /
finalizer / group_chat). The business turn_index is NEVER mutated to dodge
a collision.
"""
__tablename__ = "maf_roundtable_seat_runs"
__table_args__ = (
UniqueConstraint("maf_run_id", "agent_id", "turn_index", name="uq_maf_seat_runs_run_agent_turn"),
UniqueConstraint(
"maf_run_id",
"agent_id",
"turn_index",
"execution_stage",
name="uq_maf_seat_runs_run_agent_turn_stage",
),
# Phase-3 P1-2: enforce the closed execution_stage enum at the DB layer.
# Mirrors seat_runs.VALID_EXECUTION_STAGES — a typo (e.g. "exprt") would
# otherwise silently bypass the four-column idempotency key by creating
# a brand-new stage. This CHECK keeps both fresh creates and any raw
# migration / manual SQL honest. Keep in sync with VALID_EXECUTION_STAGES.
CheckConstraint(
"execution_stage IN ('legacy','planner','expert','format_repair',"
"'cross_review','cross_review_repair','finalizer','group_chat')",
name="ck_maf_seat_runs_execution_stage",
),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
@@ -321,12 +343,24 @@ class MafRoundtableSeatRunRow(Base):
)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
turn_index: Mapped[int] = mapped_column(Integer, nullable=False)
# Phase-3: which kind of DeerFlow execution this row represents. Pre-Phase-3
# rows are migrated to ``legacy``. Closed enum (see seat_runs.VALID_EXECUTION_STAGES):
# planner | expert | format_repair | cross_review | cross_review_repair |
# finalizer | group_chat | legacy.
execution_stage: Mapped[str] = mapped_column(
String(32), nullable=False, default="legacy", server_default="legacy"
)
deerflow_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
deerflow_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
sandbox_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
workflow_checkpoint_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# pending | running | completed | failed | cancelled
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
# Phase-3 P0-1: a sanitized, replayable snapshot of the SeatResponse written
# atomically with the completed transition. Survives a crash that happens
# after the seat run is marked completed but before the workflow checkpoint
# is persisted, so recovery never needs to re-call DeerFlow.
result_snapshot: Mapped[dict | None] = mapped_column(PortableJSON, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
@@ -12,6 +12,7 @@ from typing import Any
from uuid import uuid4
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.model import MafRoundtableSeatRunRow
@@ -21,17 +22,68 @@ def _now() -> datetime:
return datetime.now(UTC)
# ── execution_stage enum (Phase-3 P1-2) ─────────────────────────────────────
# The fixed set of distinct DeerFlow executions that may occur within one
# business round (turn_index). The four-column idempotency key
# (maf_run_id, agent_id, turn_index, execution_stage) only works if stages
# are a closed set — a typo would silently create a new stage and bypass the
# dedup. Validate on write so a misspelling raises loudly instead.
LEGACY = "legacy"
PLANNER = "planner"
EXPERT = "expert"
FORMAT_REPAIR = "format_repair"
CROSS_REVIEW = "cross_review"
CROSS_REVIEW_REPAIR = "cross_review_repair"
FINALIZER = "finalizer"
GROUP_CHAT = "group_chat"
VALID_EXECUTION_STAGES: frozenset[str] = frozenset({
LEGACY,
PLANNER,
EXPERT,
FORMAT_REPAIR,
CROSS_REVIEW,
CROSS_REVIEW_REPAIR,
FINALIZER,
GROUP_CHAT,
})
class InvalidExecutionStageError(ValueError):
"""Raised when an execution_stage outside the closed enum is written."""
class SeatRunNotFoundError(LookupError):
"""Raised when a seat run expected by the four-tuple does not exist."""
class SeatRunWaitTimeout(TimeoutError):
"""Raised when waiting for a seat run to reach a terminal state times out."""
def _validate_stage(execution_stage: str) -> str:
if execution_stage not in VALID_EXECUTION_STAGES:
raise InvalidExecutionStageError(
f"execution_stage={execution_stage!r} is not one of the allowed "
f"stages {sorted(VALID_EXECUTION_STAGES)}; a typo here would bypass "
f"the four-column idempotency key."
)
return execution_stage
def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
return {
"id": row.id,
"maf_run_id": row.maf_run_id,
"agent_id": row.agent_id,
"turn_index": row.turn_index,
"execution_stage": row.execution_stage,
"deerflow_run_id": row.deerflow_run_id,
"deerflow_thread_id": row.deerflow_thread_id,
"sandbox_id": row.sandbox_id,
"workflow_checkpoint_id": row.workflow_checkpoint_id,
"status": row.status,
"result_snapshot": row.result_snapshot,
"started_at": row.started_at.isoformat() if isinstance(row.started_at, datetime) else row.started_at,
"finished_at": row.finished_at.isoformat() if isinstance(row.finished_at, datetime) else row.finished_at,
"error": row.error,
@@ -40,6 +92,12 @@ def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
}
# Seat-run lifecycle statuses that mean "still in flight" — a second attempt to
# start the same DeerFlow execution must NOT launch another run while one of
# these exists.
_ACTIVE_STATUSES = ("pending", "running")
class MafSeatRunRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -50,14 +108,17 @@ class MafSeatRunRepository:
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str = "legacy",
deerflow_thread_id: str | None = None,
) -> dict[str, Any]:
_validate_stage(execution_stage)
now = _now()
row = MafRoundtableSeatRunRow(
id=uuid4().hex,
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
deerflow_thread_id=deerflow_thread_id,
status="pending",
created_at=now,
@@ -68,6 +129,171 @@ class MafSeatRunRepository:
await session.commit()
return _to_dict(row)
async def get_by_four_tuple(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
) -> dict[str, Any] | None:
"""Look up a seat run by its Phase-3 identity key (no DeerFlow side effects)."""
stmt = select(MafRoundtableSeatRunRow).where(
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
MafRoundtableSeatRunRow.agent_id == agent_id,
MafRoundtableSeatRunRow.turn_index == turn_index,
MafRoundtableSeatRunRow.execution_stage == execution_stage,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def wait_for_terminal_seat_run(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
timeout_seconds: float,
poll_interval_seconds: float = 0.1,
) -> dict[str, Any]:
"""Block until the seat run identified by the four-tuple reaches a terminal status.
Terminal statuses: completed, failed, cancelled, cancel_failed, cancel_unknown.
Non-terminal (still in-flight): pending, running, cancelling.
Raises:
SeatRunNotFoundError: if no row exists for the four-tuple.
SeatRunWaitTimeout: if the deadline is reached while still non-terminal.
"""
import asyncio
import time
_TERMINAL = ("completed", "failed", "cancelled", "cancel_failed", "cancel_unknown")
deadline = time.monotonic() + timeout_seconds
while True:
row = await self.get_by_four_tuple(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if row is None:
raise SeatRunNotFoundError(
f"seat run not found: maf_run_id={maf_run_id!r} agent_id={agent_id!r} "
f"turn_index={turn_index} execution_stage={execution_stage!r}"
)
if row["status"] in _TERMINAL:
return row
if time.monotonic() >= deadline:
raise SeatRunWaitTimeout(
f"timed out waiting for seat run {row['id']!r} "
f"(stage={execution_stage!r}, status={row['status']!r}) "
f"after {timeout_seconds}s"
)
await asyncio.sleep(poll_interval_seconds)
async def get_or_create_seat_run(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
deerflow_thread_id: str | None = None,
) -> tuple[dict[str, Any], bool]:
"""Atomic idempotent seat-run acquisition (Phase-3 P0-1).
Race-free: tries to INSERT first; on a four-column unique-key conflict
it rolls back the short transaction and reads the existing row. This is
strictly stronger than "query-then-insert" because two concurrent
callers that both miss the lookup would otherwise both insert — only
the DB constraint decides the winner, and the loser reads the winner's
row.
Because the winner's transaction may not have committed by the time the
loser's post-conflict read runs (session isolation), the loser retries
the read a bounded number of times with a short backoff before giving
up. ``IntegrityError`` is NEVER propagated to the caller on a four-tuple
conflict.
Returns ``(seat_run, created)``:
- ``created=True`` → caller OWNS this execution and may start DeerFlow.
- ``created=False`` → a row already exists; the caller must inspect its
``status`` and must NOT start a second DeerFlow run for it.
"""
_validate_stage(execution_stage)
now = _now()
row = MafRoundtableSeatRunRow(
id=uuid4().hex,
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
deerflow_thread_id=deerflow_thread_id,
status="pending",
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError:
# Lost the race (or replayed an existing four-tuple). Roll back
# this short transaction, then read the winner's row. The winner
# may not have committed yet (session isolation), so retry a few
# times with a short backoff before treating this as a hard
# error.
await session.rollback()
existing = await self._await_existing(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if existing is None:
# The conflict was NOT on our four-tuple (e.g. a FK
# violation) — surface it as a clean error rather than
# swallowing it.
raise
return existing, False
return _to_dict(row), True
async def _await_existing(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
retries: int = 10,
backoff_s: float = 0.01,
) -> dict[str, Any] | None:
"""Read the winner's row, retrying briefly until it is visible.
Under true concurrency the winner's COMMIT may land after the loser's
post-rollback SELECT; the loser cannot see an uncommitted row, so we
poll with a tiny backoff until the winner's transaction commits (or the
retry budget is exhausted, in which case we return None so the caller
can decide it was a non-four-tuple conflict).
"""
import asyncio
for _ in range(max(1, retries)):
existing = await self.get_by_four_tuple(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if existing is not None:
return existing
await asyncio.sleep(backoff_s)
return None
async def mark_running(
self, seat_run_id: str, *, deerflow_run_id: str, sandbox_id: str | None = None
) -> None:
@@ -85,17 +311,29 @@ class MafSeatRunRepository:
)
await session.commit()
async def mark_completed(self, seat_run_id: str, *, checkpoint_id: str | None = None) -> None:
async def mark_completed(
self,
seat_run_id: str,
*,
checkpoint_id: str | None = None,
result_snapshot: dict[str, Any] | None = None,
) -> None:
"""Mark a seat run completed, optionally persisting a sanitized result
snapshot so it can be replayed after a crash even if the workflow
checkpoint was not yet written."""
values: dict[str, Any] = {
"status": "completed",
"workflow_checkpoint_id": checkpoint_id,
"finished_at": _now(),
"updated_at": _now(),
}
if result_snapshot is not None:
values["result_snapshot"] = result_snapshot
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(
status="completed",
workflow_checkpoint_id=checkpoint_id,
finished_at=_now(),
updated_at=_now(),
)
.values(**values)
)
await session.commit()
@@ -163,7 +401,10 @@ class MafSeatRunRepository:
stmt = (
select(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.maf_run_id == maf_run_id)
.order_by(MafRoundtableSeatRunRow.turn_index.asc())
.order_by(
MafRoundtableSeatRunRow.turn_index.asc(),
MafRoundtableSeatRunRow.execution_stage.asc(),
)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()