feat(maf): add secure roundtable workflow runtime

This commit is contained in:
董宏凯 committed 2026-08-01 17:51:53 +08:00
1 parent e690475e6e
commit bd4131b4e2
52 files changed
+10315 -96

No files matched your search

@@ -13,47 +13,57 @@ started (Phase 2+).
Tables (see the implementation plan §8):
- ``maf_roundtable_sessions`` — one MAF roundtable session (task + mode + capability snapshot)
- ``maf_roundtable_participants`` — manager + seat agents bound to a session
- ``maf_roundtable_runs`` — one execution run of a session
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
- ``maf_roundtable_approvals`` — pending human-approval requests
- ``maf_roundtable_sessions`` — one MAF roundtable session (task + mode + capability snapshot)
- ``maf_roundtable_participants`` — manager + seat agents bound to a session
- ``maf_roundtable_runs`` — one execution run of a session
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
- ``maf_roundtable_approvals`` — pending human-approval requests
- ``maf_roundtable_seat_runs`` — MAF↔DeerFlow run mapping (survives restart)
- ``maf_roundtable_action_secrets`` — encrypted actual execution params (server-only)
"""
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.action_secrets import MafActionSecretRepository
from deerflow.persistence.maf_roundtable.actions import MafActionRepository
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
from deerflow.persistence.maf_roundtable.events import MafEventRepository
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableActionRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
MafRoundtableEventRow,
MafRoundtableParticipantRow,
MafRoundtableRunRow,
MafRoundtableSeatRunRow,
MafRoundtableSessionRow,
)
from deerflow.persistence.maf_roundtable.participants import MafParticipantRepository
from deerflow.persistence.maf_roundtable.runs import MafRunRepository
from deerflow.persistence.maf_roundtable.seat_runs import MafSeatRunRepository
from deerflow.persistence.maf_roundtable.sessions import MafSessionRepository
from deerflow.persistence.maf_roundtable.store import MafRoundtableStore
__all__ = [
"MafActionRepository",
"MafActionSecretRepository",
"MafApprovalRepository",
"MafEventRepository",
"MafParticipantRepository",
"MafRoundtableActionRow",
"MafRoundtableActionSecretRow",
"MafRoundtableApprovalRow",
"MafRoundtableEventRow",
"MafRoundtableParticipantRow",
"MafRoundtableRunRow",
"MafRoundtableSeatRunRow",
"MafRoundtableSessionRow",
"MafRoundtableStore",
"MafRunRepository",
"MafSeatRunRepository",
"MafSessionRepository",
"make_maf_roundtable_store",
]
@@ -0,0 +1,65 @@
"""Action secrets repository — encrypted actual execution parameters.
Stores the **real** (un-redacted) tool-call arguments for a protected action
so that after approval or a service restart the system can recover and
execute the approved action with the original parameters. This repository is
**server-only**: it is never read by any API endpoint, event serializer, log
formatter, or frontend-facing query. The ActionCoordinator encrypts before
writing and decrypts only at execution time.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.model import MafRoundtableActionSecretRow
def _now() -> datetime:
return datetime.now(UTC)
class MafActionSecretRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create(self, *, action_id: str, arguments_encrypted: str) -> dict[str, Any]:
now = _now()
row = MafRoundtableActionSecretRow(
id=uuid4().hex,
action_id=action_id,
arguments_encrypted=arguments_encrypted,
created_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
return {"id": row.id, "action_id": action_id}
async def get_by_action(self, action_id: str) -> str | None:
"""Return the encrypted arguments ciphertext for an action, or None."""
async with self._sf() as session:
stmt = (
select(MafRoundtableActionSecretRow)
.where(MafRoundtableActionSecretRow.action_id == action_id)
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
return row.arguments_encrypted if row is not None else None
async def delete_by_action(self, action_id: str) -> None:
async with self._sf() as session:
stmt = (
select(MafRoundtableActionSecretRow)
.where(MafRoundtableActionSecretRow.action_id == action_id)
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is not None:
await session.delete(row)
await session.commit()
@@ -14,7 +14,7 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.model import MafRoundtableActionRow
@@ -24,7 +24,8 @@ def _now() -> datetime:
return datetime.now(UTC)
def _to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
def action_to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
"""Serialise an action ORM row to a plain dict (public — reused by the store facade)."""
return {
"id": row.id,
"session_id": row.session_id,
@@ -32,6 +33,7 @@ def _to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
"agent_id": row.agent_id,
"tool_name": row.tool_name,
"logical_action_id": row.logical_action_id,
"turn_index": row.turn_index,
"idempotency_key": row.idempotency_key,
"target_key": row.target_key,
"risk_level": row.risk_level,
@@ -48,6 +50,25 @@ def _to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
}
# Back-compat alias for internal callers that used the old private name.
_to_dict = action_to_dict
class MafActionStateMachineError(RuntimeError):
"""An action was claimed or transitioned from an unexpected state.
Raised when a claim loses the ``approved → running`` CAS and the row is in
a state that is neither ``running`` (in progress elsewhere) nor a recognized
terminal (``succeeded`` / ``failed`` / ``unknown`` / ``rejected``) — e.g.
still ``waiting_approval`` / ``pending`` — meaning the caller tried to
execute an action that was never properly approved.
"""
# Terminal statuses a failed claim may reflect back to the caller.
_TERMINAL_CLAIM_STATUSES = ("succeeded", "failed", "unknown", "rejected")
class MafActionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -65,6 +86,7 @@ class MafActionRepository:
agent_id=data.get("agent_id"),
tool_name=str(data["tool_name"]),
logical_action_id=data.get("logical_action_id"),
turn_index=data.get("turn_index"),
idempotency_key=str(data["idempotency_key"]),
target_key=data.get("target_key"),
risk_level=str(data.get("risk_level") or "R0"),
@@ -135,3 +157,131 @@ class MafActionRepository:
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def claim_for_execution(self, action_id: str) -> str:
"""Atomically transition an action from ``approved`` to ``running``.
Single conditional UPDATE — only the caller whose UPDATE affects
exactly one row wins the right to execute the external write. This
guarantees at-most-once execution even under concurrent ``resume()``
calls or retries after a restart.
Returns the effective claim status (callers must NOT treat a lost claim
as a blanket "skip"):
- ``"claimed"`` — won the CAS; the caller may execute.
- ``"in_progress"`` — another executor already moved it to ``running``.
- ``"succeeded"`` / ``"failed"`` / ``"unknown"`` / ``"rejected"`` — the
action is already in that terminal state.
Raises ``MafActionStateMachineError`` when the row is missing or in any
other state (e.g. still ``waiting_approval`` / ``pending``) — i.e. it was
never properly approved and must not be executed.
"""
now = _now()
async with self._sf() as session:
result = await session.execute(
update(MafRoundtableActionRow)
.where(
MafRoundtableActionRow.id == action_id,
MafRoundtableActionRow.status == "approved",
)
.values(status="running", started_at=now, updated_at=now)
)
await session.commit()
if result.rowcount == 1: # type: ignore[union-attr]
return "claimed"
# Lost the CAS (or never approved) — report the ACTUAL state.
current = await self.get_action(action_id)
if current is None:
raise MafActionStateMachineError(f"action {action_id} not found during claim")
status = current.get("status")
if status == "running":
return "in_progress"
if status in _TERMINAL_CLAIM_STATUSES:
return str(status)
raise MafActionStateMachineError(
f"action {action_id} cannot be claimed from state '{status}'"
)
async def mark_approved(self, action_id: str) -> bool:
"""Conditionally transition ``waiting_approval → approved``.
Returns True only if this call performed the transition (``rowcount==1``).
A concurrent decider or an already-terminal action yields False — the
caller must NOT overwrite a terminal state via a generic update.
"""
now = _now()
async with self._sf() as session:
result = await session.execute(
update(MafRoundtableActionRow)
.where(
MafRoundtableActionRow.id == action_id,
MafRoundtableActionRow.status == "waiting_approval",
)
.values(status="approved", updated_at=now)
)
await session.commit()
return result.rowcount == 1 # type: ignore[union-attr]
async def mark_rejected(self, action_id: str) -> bool:
"""Conditionally transition ``waiting_approval → rejected`` (+ finished_at).
Returns True only if this call performed the transition. Terminal states
are never overwritten.
"""
now = _now()
async with self._sf() as session:
result = await session.execute(
update(MafRoundtableActionRow)
.where(
MafRoundtableActionRow.id == action_id,
MafRoundtableActionRow.status == "waiting_approval",
)
.values(status="rejected", finished_at=now, updated_at=now)
)
await session.commit()
return result.rowcount == 1 # type: ignore[union-attr]
async def record_terminal_result(
self,
action_id: str,
*,
status: str,
result: dict[str, Any] | None = None,
external_object_id: str | None = None,
) -> dict[str, Any] | None:
"""Record the terminal outcome of an action execution.
State-machine constrained: only a ``running`` action may transition
to a terminal status (``succeeded`` / ``failed`` / ``unknown``).
A concurrent caller that lost the race (or a replay after restart)
finds the row already terminal and its UPDATE is a no-op
(``rowcount == 0``) — it must NOT overwrite the real result.
"""
if status not in ("succeeded", "failed", "unknown"):
raise ValueError(f"invalid action result status: {status}")
now = _now()
async with self._sf() as session:
result_q = await session.execute(
update(MafRoundtableActionRow)
.where(
MafRoundtableActionRow.id == action_id,
MafRoundtableActionRow.status == "running",
)
.values(
status=status,
result_redacted_json=result,
external_object_id=external_object_id,
finished_at=now,
updated_at=now,
)
)
await session.commit()
# Use rowcount instead of .returning() for cross-DB compatibility
# (SQLite, PostgreSQL, MySQL).
if result_q.rowcount != 1: # type: ignore[union-attr]
# Lost the race or not in running state — do not overwrite.
return None
return await self.get_action(action_id)
@@ -46,6 +46,7 @@ def _to_dict(row: MafRoundtableEventRow) -> dict[str, Any]:
"event_type": row.event_type,
"agent_id": row.agent_id,
"turn_index": row.turn_index,
"source_event_id": row.source_event_id,
"payload": row.payload_json,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
}
@@ -69,6 +70,7 @@ class MafEventRepository:
agent_id: str | None = None,
turn_index: int | None = None,
payload: dict[str, Any] | None = None,
source_event_id: str | None = None,
) -> dict[str, Any]:
"""Append one event with a monotonic per-run seq.
@@ -77,7 +79,19 @@ class MafEventRepository:
``last_event_seq`` cache is bumped forward (max wins) so pollers can
cheaply check ``runs.last_event_seq`` without scanning the events
table.
Durable dedup: when ``source_event_id`` is given, the
``(run_id, source_event_id)`` unique constraint guarantees at-most-once
persistence. A duplicate insert returns the already-stored row instead
of raising — this survives process restarts (unlike an in-memory seen
set), so a replayed runtime stream never double-writes an event.
"""
# Fast-path dedup: if this source_event_id already landed, return it.
if source_event_id:
existing = await self._get_by_source(run_id, source_event_id)
if existing is not None:
return existing
last_exc: Exception | None = None
for _ in range(_SEQ_INSERT_RETRIES):
async with self._sf() as session:
@@ -97,6 +111,7 @@ class MafEventRepository:
event_type=event_type,
agent_id=agent_id,
turn_index=turn_index,
source_event_id=source_event_id,
payload_json=payload,
created_at=_now(),
)
@@ -104,8 +119,14 @@ class MafEventRepository:
try:
await session.commit()
except IntegrityError as exc: # noqa: PERF203 - retry path
last_exc = exc
await session.rollback()
# A source_event_id collision means a concurrent writer
# already persisted this exact event — return its row.
if source_event_id:
dup = await self._get_by_source(run_id, source_event_id)
if dup is not None:
return dup
last_exc = exc
continue
if self._run_repo is not None:
# best-effort cache bump; never let a cache failure abort
@@ -121,6 +142,15 @@ class MafEventRepository:
orig=last_exc,
)
async def _get_by_source(self, run_id: str, source_event_id: str) -> dict[str, Any] | None:
stmt = select(MafRoundtableEventRow).where(
MafRoundtableEventRow.run_id == run_id,
MafRoundtableEventRow.source_event_id == source_event_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalars().first()
return _to_dict(row) if row is not None else None
async def append_events(
self, *, session_id: str, run_id: str, events: list[dict[str, Any]]
) -> list[dict[str, Any]]:
@@ -138,6 +168,7 @@ class MafEventRepository:
agent_id=ev.get("agent_id"),
turn_index=ev.get("turn_index"),
payload=ev.get("payload"),
source_event_id=ev.get("source_event_id"),
)
)
return out
@@ -175,6 +175,13 @@ class MafRoundtableEventRow(Base):
__tablename__ = "maf_roundtable_events"
__table_args__ = (
UniqueConstraint("run_id", "seq", name="uq_maf_events_run_seq"),
# Dedup key for at-most-once event persistence. The authoritative
# dedup is this DB constraint (survives process restarts) — not the
# in-process ``_seen_source_ids`` set, which is only a fast-path cache.
# ``source_event_id`` is nullable; rows with NULL are not deduped
# (SQL treats NULLs as distinct in a UNIQUE constraint on sqlite /
# postgres / mysql), so events without a source id stay append-only.
UniqueConstraint("run_id", "source_event_id", name="uq_maf_events_run_source"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
@@ -194,6 +201,10 @@ class MafRoundtableEventRow(Base):
event_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
turn_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Upstream identity of the source runtime event (e.g. LangGraph message id
# / tool_call id). Used by the ``(run_id, source_event_id)`` unique
# constraint for durable at-most-once dedup across restarts.
source_event_id: Mapped[str | None] = mapped_column(String(191), nullable=True, index=True)
payload_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, index=True)
@@ -227,6 +238,11 @@ class MafRoundtableActionRow(Base):
agent_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
tool_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
logical_action_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# The seat turn that produced this action. Persisted (not only embedded in
# the idempotency_key) so pending-approval matching can key on
# run_id + agent_id + turn_index + logical_action_id precisely — never
# mixing a seat's earlier-turn waiting actions into the current approval.
turn_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
idempotency_key: Mapped[str] = mapped_column(String(191), nullable=False, unique=True)
target_key: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
# R0 | R1 | R2 | R3 | R4
@@ -280,11 +296,84 @@ class MafRoundtableApprovalRow(Base):
)
# ── 7. seat runs (Phase 2: MAF↔DeerFlow run mapping) ─────────────────────
class MafRoundtableSeatRunRow(Base):
"""Persistent mapping between a MAF workflow seat turn and its DeerFlow run.
Replaces the in-memory ``_active_runs`` dict so that after a service restart
the system can still locate and cancel outstanding DeerFlow runs, and the
MAF checkpoint can reference which seat turns completed.
Unique constraint: ``(maf_run_id, agent_id, turn_index)`` — one DeerFlow run
per seat per turn.
"""
__tablename__ = "maf_roundtable_seat_runs"
__table_args__ = (
UniqueConstraint("maf_run_id", "agent_id", "turn_index", name="uq_maf_seat_runs_run_agent_turn"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
maf_run_id: Mapped[str] = mapped_column(
String(64), ForeignKey("maf_roundtable_runs.id", ondelete="CASCADE"), nullable=False, index=True
)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
turn_index: Mapped[int] = mapped_column(Integer, nullable=False)
deerflow_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
deerflow_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
sandbox_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
workflow_checkpoint_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# pending | running | completed | failed | cancelled
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
)
# ── 8. action secrets (encrypted actual execution params, server-only) ────
class MafRoundtableActionSecretRow(Base):
"""Encrypted actual execution parameters for a protected action.
Stores the **real** (un-redacted) tool-call arguments so that after
approval or a service restart the system can recover and execute the
approved action with the original parameters. This table is **never**
joined in any read query exposed to the API, events, logs, or frontend —
it is server-only. The ActionCoordinator encrypts the payload before
writing and decrypts only at execution time.
One row per action (unique ``action_id``). The encryption key is derived
from the deployment's secret key; the ciphertext is opaque to the store.
"""
__tablename__ = "maf_roundtable_action_secrets"
__table_args__ = (
UniqueConstraint("action_id", name="uq_maf_action_secrets_action"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
action_id: Mapped[str] = mapped_column(
String(64), ForeignKey("maf_roundtable_actions.id", ondelete="CASCADE"),
nullable=False, index=True,
)
arguments_encrypted: Mapped[str] = mapped_column(PortableLongText, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
__all__ = [
"MafRoundtableActionRow",
"MafRoundtableActionSecretRow",
"MafRoundtableApprovalRow",
"MafRoundtableEventRow",
"MafRoundtableParticipantRow",
"MafRoundtableRunRow",
"MafRoundtableSeatRunRow",
"MafRoundtableSessionRow",
]
@@ -120,3 +120,25 @@ class MafParticipantRepository:
row.updated_at = _now()
await session.commit()
return _to_dict(row)
async def bind_thread(
self, session_id: str, agent_id: str, *, thread_id: str, sandbox_id: str | None = None
) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = (
select(MafRoundtableParticipantRow)
.where(
MafRoundtableParticipantRow.session_id == session_id,
MafRoundtableParticipantRow.agent_id == agent_id,
)
.limit(1)
)
row = (await session.execute(stmt)).scalars().first()
if row is None:
return None
row.thread_id = thread_id
if sandbox_id is not None:
row.sandbox_id = sandbox_id
row.updated_at = _now()
await session.commit()
return _to_dict(row)
@@ -118,6 +118,27 @@ class MafRunRepository:
)
await session.commit()
async def claim_for_resume(self, run_id: str) -> bool:
"""Run-level CAS gating post-approval execution: ``waiting_approval → running``.
After every approval is decided, exactly one concurrent ``resume()`` may
proceed to execute the approved actions and write the parent run's final
state. The winner (``rowcount == 1``) executes; every other concurrent
caller gets False and must return the current run WITHOUT writing a
terminal status (so it can never race a ``completed``/``failed`` write).
"""
async with self._sf() as session:
result = await session.execute(
update(MafRoundtableRunRow)
.where(
MafRoundtableRunRow.id == run_id,
MafRoundtableRunRow.status == "waiting_approval",
)
.values(status="running", started_at=_now(), updated_at=_now())
)
await session.commit()
return result.rowcount == 1 # type: ignore[union-attr]
async def bump_last_event_seq(self, run_id: str, seq: int) -> None:
"""Denormalized cache update — only bumps forward (max wins)."""
async with self._sf() as session:
@@ -0,0 +1,195 @@
"""Seat run repository — persistent MAF↔DeerFlow run mapping.
Replaces the in-memory ``_active_runs`` dict. After a service restart the
system can still locate and cancel outstanding DeerFlow runs via the
persisted ``deerflow_run_id``.
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.model import MafRoundtableSeatRunRow
def _now() -> datetime:
return datetime.now(UTC)
def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
return {
"id": row.id,
"maf_run_id": row.maf_run_id,
"agent_id": row.agent_id,
"turn_index": row.turn_index,
"deerflow_run_id": row.deerflow_run_id,
"deerflow_thread_id": row.deerflow_thread_id,
"sandbox_id": row.sandbox_id,
"workflow_checkpoint_id": row.workflow_checkpoint_id,
"status": row.status,
"started_at": row.started_at.isoformat() if isinstance(row.started_at, datetime) else row.started_at,
"finished_at": row.finished_at.isoformat() if isinstance(row.finished_at, datetime) else row.finished_at,
"error": row.error,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
class MafSeatRunRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_seat_run(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
deerflow_thread_id: str | None = None,
) -> dict[str, Any]:
now = _now()
row = MafRoundtableSeatRunRow(
id=uuid4().hex,
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
deerflow_thread_id=deerflow_thread_id,
status="pending",
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await session.commit()
return _to_dict(row)
async def mark_running(
self, seat_run_id: str, *, deerflow_run_id: str, sandbox_id: str | None = None
) -> None:
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(
status="running",
deerflow_run_id=deerflow_run_id,
sandbox_id=sandbox_id,
started_at=_now(),
updated_at=_now(),
)
)
await session.commit()
async def mark_completed(self, seat_run_id: str, *, checkpoint_id: str | None = None) -> None:
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(
status="completed",
workflow_checkpoint_id=checkpoint_id,
finished_at=_now(),
updated_at=_now(),
)
)
await session.commit()
async def mark_failed(self, seat_run_id: str, *, error: str | None = None) -> None:
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(status="failed", error=error, finished_at=_now(), updated_at=_now())
)
await session.commit()
async def mark_cancelled(self, seat_run_id: str) -> None:
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(status="cancelled", finished_at=_now(), updated_at=_now())
)
await session.commit()
async def mark_cancelling(self, seat_run_id: str) -> None:
"""Mark a seat run as cancelling (cancel requested, not yet confirmed)."""
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(status="cancelling", updated_at=_now())
)
await session.commit()
async def mark_cancel_failed(self, seat_run_id: str, *, error: str | None = None) -> None:
"""Mark a seat run as cancel_failed (DeerFlow cancel did not succeed)."""
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(status="cancel_failed", error=error, updated_at=_now())
)
await session.commit()
async def mark_cancel_unknown(self, seat_run_id: str, *, error: str | None = None) -> None:
"""Mark a seat run's cancel outcome as unknown (e.g. no DeerFlow run id)."""
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(status="cancel_unknown", error=error, updated_at=_now())
)
await session.commit()
async def get_active_by_maf_run(self, maf_run_id: str) -> list[dict[str, Any]]:
stmt = (
select(MafRoundtableSeatRunRow)
.where(
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
MafRoundtableSeatRunRow.status.in_(("pending", "running")),
)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [_to_dict(r) for r in rows]
async def get_by_maf_run(self, maf_run_id: str) -> list[dict[str, Any]]:
stmt = (
select(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.maf_run_id == maf_run_id)
.order_by(MafRoundtableSeatRunRow.turn_index.asc())
)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [_to_dict(r) for r in rows]
async def cancel_all_active(self, maf_run_id: str) -> list[dict[str, Any]]:
"""Mark all active seat runs as cancelling and return their info.
Two-phase cancel: first mark as ``cancelling``, then the caller
actually cancels the DeerFlow runs and calls ``mark_cancelled`` or
``mark_cancel_failed`` per seat run. Returns list of
``{seat_run_id, deerflow_run_id}`` dicts.
"""
stmt = (
select(MafRoundtableSeatRunRow)
.where(
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
MafRoundtableSeatRunRow.status.in_(("pending", "running")),
)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
result = []
for row in rows:
row.status = "cancelling"
row.updated_at = _now()
result.append({"seat_run_id": row.id, "deerflow_run_id": row.deerflow_run_id})
await session.commit()
return result
@@ -18,21 +18,31 @@ from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
from sqlalchemy import select, update
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.actions import MafActionRepository
from deerflow.persistence.maf_roundtable.action_secrets import MafActionSecretRepository
from deerflow.persistence.maf_roundtable.actions import MafActionRepository, action_to_dict
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
from deerflow.persistence.maf_roundtable.events import MafEventRepository
from deerflow.persistence.maf_roundtable.model import MafRoundtableRunRow, MafRoundtableSessionRow
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableActionRow,
MafRoundtableActionSecretRow,
MafRoundtableApprovalRow,
MafRoundtableEventRow,
MafRoundtableRunRow,
MafRoundtableSessionRow,
)
from deerflow.persistence.maf_roundtable.participants import MafParticipantRepository
from deerflow.persistence.maf_roundtable.runs import MafRunRepository, run_to_dict
from deerflow.persistence.maf_roundtable.seat_runs import MafSeatRunRepository
from deerflow.persistence.maf_roundtable.sessions import MafSessionRepository
# Session statuses that mean a run is in flight — the CAS claim refuses these.
_ACTIVE_SESSION_STATUSES = ("starting", "running", "waiting_approval", "waiting_input")
_ACTIVE_SESSION_STATUSES = ("starting", "running", "waiting_approval", "waiting_input", "cancelling", "cancel_failed")
# Run statuses that are still in flight (cancellable).
_ACTIVE_RUN_STATUSES = ("pending", "running", "waiting_approval", "waiting_input")
_ACTIVE_RUN_STATUSES = ("pending", "running", "waiting_approval", "waiting_input", "cancelling", "cancel_failed")
# Run statuses that are terminal (cannot be re-cancelled).
_TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
@@ -57,6 +67,8 @@ class MafRoundtableStore:
self.events = MafEventRepository(session_factory, run_repo=self.runs)
self.actions = MafActionRepository(session_factory)
self.approvals = MafApprovalRepository(session_factory)
self.seat_runs = MafSeatRunRepository(session_factory)
self.action_secrets = MafActionSecretRepository(session_factory)
@property
def session_factory(self) -> async_sessionmaker[AsyncSession]:
@@ -160,13 +172,15 @@ class MafRoundtableStore:
if run_row.status in _ACTIVE_RUN_STATUSES:
# Normal cancel: run + session → cancelled.
# Also handles cancel_failed → cancelled retry.
run_row.status = "cancelled"
run_row.finished_at = now
run_row.updated_at = now
session_row.status = "cancelled"
session_row.updated_at = now
elif run_row.status == "cancelled":
# Already cancelled — repair session if still active.
elif run_row.status == "cancelled" or run_row.status == "cancel_failed":
# Already cancelled / previously failed cancel — repair
# session if still active.
if session_row.status in _ACTIVE_SESSION_STATUSES:
session_row.status = "cancelled"
session_row.updated_at = now
@@ -180,9 +194,302 @@ class MafRoundtableStore:
await db.flush()
return run_to_dict(run_row)
async def start_cancel_atomically(
self,
*,
run_id: str,
owner_user_id: str,
) -> dict[str, Any] | None:
"""Atomically transition a run to ``cancelling`` (not yet ``cancelled``).
Phase 1 of the two-phase cancel cascade:
1. Router calls this → run+session enter ``cancelling``.
2. Router calls ``dispatcher.cancel(run_id)`` which cascades to
DeerFlow sub-runs.
3. After the cascade, ``cancel_run_atomically`` finalizes to
``cancelled``.
Returns the updated run dict, or None if the run is missing,
wrong-owner, or already terminal.
"""
now = _now()
async with self._sf() as db:
async with db.begin():
stmt = (
select(MafRoundtableRunRow, MafRoundtableSessionRow)
.join(
MafRoundtableSessionRow,
MafRoundtableRunRow.session_id == MafRoundtableSessionRow.id,
)
.where(
MafRoundtableRunRow.id == run_id,
MafRoundtableSessionRow.user_id == owner_user_id,
)
.with_for_update()
)
row = (await db.execute(stmt)).first()
if row is None:
return None
run_row, session_row = row
# Transition ANY active (or cancel_failed retry) run to cancelling.
# Includes: pending (not yet executing), running, waiting_*,
# cancelling (no-op, return current), cancel_failed (retry).
if run_row.status in _ACTIVE_RUN_STATUSES and run_row.status != "cancelling":
run_row.status = "cancelling"
run_row.updated_at = now
session_row.status = "cancelling"
session_row.updated_at = now
await db.flush()
return run_to_dict(run_row)
# Already cancelling or terminal — return current state.
return run_to_dict(run_row)
async def mark_cancel_failed_atomically(
self,
*,
run_id: str,
owner_user_id: str,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
"""Atomically mark run+session as ``cancel_failed``.
Unlike ``mark_cancel_failed`` in service.py (which only updates run),
this writes run AND session in a single transaction so their statuses
never diverge. ``cancel_failed`` is a non-terminal state: the user
can retry cancel later.
"""
now = _now()
async with self._sf() as db:
async with db.begin():
stmt = (
select(MafRoundtableRunRow, MafRoundtableSessionRow)
.join(
MafRoundtableSessionRow,
MafRoundtableRunRow.session_id == MafRoundtableSessionRow.id,
)
.where(
MafRoundtableRunRow.id == run_id,
MafRoundtableSessionRow.user_id == owner_user_id,
)
.with_for_update()
)
row = (await db.execute(stmt)).first()
if row is None:
return None
run_row, session_row = row
if run_row.status not in ("cancelling", "cancel_failed"):
return run_to_dict(run_row)
run_row.status = "cancel_failed"
run_row.updated_at = now
if error_code:
run_row.error_code = error_code
if error_message:
run_row.error_message_safe = error_message
session_row.status = "cancel_failed"
session_row.updated_at = now
await db.flush()
return run_to_dict(run_row)
async def create_pending_action_bundle(
self,
*,
session_id: str,
run_id: str,
agent_id: str,
tool_name: str,
logical_action_id: str | None,
idempotency_key: str,
risk_level: str,
target_key: str | None,
turn_index: int | None = None,
arguments_sanitized: dict[str, Any] | None,
arguments_encrypted: str,
approval_summary: str,
approval_payload: dict[str, Any] | None,
audit_event_agent_id: str,
audit_event_turn_index: int,
) -> dict[str, Any]:
"""Create an action + secret + approval + audit event in ONE transaction.
All four writes share a single ``AsyncSession`` + ``begin()`` so that
any failure rolls everything back — the database never holds a
half-finished action (action without secret / approval / event).
Idempotency: ``idempotency_key`` is DB-unique. A concurrent duplicate
submission loses the INSERT (``IntegrityError``); the method then
re-reads and returns the already-stored action instead of raising,
so a retry never fails the run.
**P1-6 concurrent retry**: when multiple parallel seat turns create
action bundles concurrently, the ``MAX(seq)+1`` approach may collide
on the ``(run_id, seq)`` unique constraint. This method retries up to
``_MAX_BUNDLE_RETRIES`` times, re-computing ``next_seq`` from the
latest committed state inside a fresh transaction each time.
Returns the stored action dict (with ``approval_id`` linked). The
returned dict carries a ``"_idempotent_replay"`` flag (True when an
existing action was returned rather than newly created).
"""
_MAX_BUNDLE_RETRIES = 8
# Idempotency fast-path: check if an identical action already exists
# (concurrent retry / replay after restart). This check is outside the
# retry loop because idempotency key duplicates are independent of seq
# collisions.
async with self._sf() as db:
existing = await db.execute(
select(MafRoundtableActionRow).where(
MafRoundtableActionRow.idempotency_key == idempotency_key
).limit(1)
)
existing_row = existing.scalars().first()
if existing_row is not None:
d = action_to_dict(existing_row)
d["_idempotent_replay"] = True
return d
now = _now()
action_id = uuid4().hex
last_error: Exception | None = None
for attempt in range(1, _MAX_BUNDLE_RETRIES + 1):
async with self._sf() as db:
async with db.begin():
# Re-check idempotency inside transaction (concurrent writer
# may have committed between attempts).
existing = await db.execute(
select(MafRoundtableActionRow).where(
MafRoundtableActionRow.idempotency_key == idempotency_key
).limit(1)
)
existing_row2 = existing.scalars().first()
if existing_row2 is not None:
d = action_to_dict(existing_row2)
d["_idempotent_replay"] = True
return d
# 1. action row.
action_row = MafRoundtableActionRow(
id=action_id,
session_id=session_id,
run_id=run_id,
agent_id=agent_id,
tool_name=tool_name,
logical_action_id=logical_action_id,
idempotency_key=idempotency_key,
target_key=target_key,
turn_index=turn_index,
risk_level=risk_level,
status="waiting_approval",
arguments_redacted_json=arguments_sanitized,
created_at=now,
updated_at=now,
)
db.add(action_row)
try:
await db.flush()
except IntegrityError:
last_error = IntegrityError("action row insert conflict", orig=None, params={})
await db.rollback()
# Re-read — another writer may have created the action.
async with self._sf() as db2:
existing3 = await db2.execute(
select(MafRoundtableActionRow).where(
MafRoundtableActionRow.idempotency_key == idempotency_key
).limit(1)
)
er = existing3.scalars().first()
if er is not None:
d = action_to_dict(er)
d["_idempotent_replay"] = True
return d
continue
# 2. encrypted secret (server-only).
db.add(MafRoundtableActionSecretRow(
id=uuid4().hex,
action_id=action_id,
arguments_encrypted=arguments_encrypted,
created_at=now,
))
await db.flush()
# 3. approval row (pending).
approval_id = uuid4().hex
db.add(MafRoundtableApprovalRow(
id=approval_id,
session_id=session_id,
run_id=run_id,
action_id=action_id,
requested_by_agent_id=agent_id,
status="pending",
summary=approval_summary,
payload_redacted_json=approval_payload,
created_at=now,
updated_at=now,
))
await db.flush()
# 4. link approval back to action.
action_row.approval_id = approval_id
action_row.updated_at = now
await db.flush()
# 5. critical audit event — action.waiting_approval.
# Recompute seq fresh inside each attempt/transaction.
max_seq = (
await db.execute(
select(func.max(MafRoundtableEventRow.seq)).where(
MafRoundtableEventRow.run_id == run_id
)
)
).scalar_one()
next_seq = int(max_seq or 0) + 1
db.add(MafRoundtableEventRow(
id=uuid4().hex,
session_id=session_id,
run_id=run_id,
seq=next_seq,
event_type="action.waiting_approval",
agent_id=audit_event_agent_id,
turn_index=audit_event_turn_index,
payload_json={
"action_id": action_id,
"approval_id": approval_id,
"tool_name": tool_name,
"risk_level": risk_level,
},
created_at=now,
))
try:
await db.flush()
except IntegrityError:
# (run_id, seq) collision — retry with fresh seq.
last_error = IntegrityError("event seq collision", orig=None, params={})
await db.rollback()
continue
result = action_to_dict(action_row)
result["approval_id"] = approval_id
result["_idempotent_replay"] = False
return result
raise RuntimeError(
f"create_pending_action_bundle failed after {_MAX_BUNDLE_RETRIES} attempts "
f"for idempotency_key={idempotency_key}"
) from last_error
__all__ = [
"MafActionRepository",
"MafActionSecretRepository",
"MafApprovalRepository",
"MafEventRepository",
"MafParticipantRepository",