feat(maf): Phase 5 - action coordinator, approval flow, frontend MAF roundtable pages
- Backend: action_coordinator, capability_snapshot, event_normalizer, tool_executor updates - Backend: approved_tool_executor with write guard, action leases, migration script - Frontend: maf-roundtable module (API client, event timeline, approval cards, fact flow panel) - Frontend: MafRoutes, MafSessionsPage, MafRunDetailPage - Tests: action_leases, migration_phase5, phase5_entry_e2e, real_write_e2e/http_e2e - Docs: update implementation plan Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
21c4c898e7
commit
f82363d81f
55 files changed
+6304
-167
No files matched your search
+6
-3
@@ -1,9 +1,9 @@
|
||||
"""Microsoft Agent Framework (MAF) roundtable experiment persistence.
|
||||
|
||||
Six independent tables — **no foreign keys to the legacy ``roundtable_*``
|
||||
Nine independent tables — **no foreign keys to the legacy ``roundtable_*``
|
||||
tables** — so the existing multi-agent roundtable keeps its current behavior,
|
||||
schema, and recovery semantics untouched. The MAF experiment is fully
|
||||
isolated: it has its own session / participant / run / event / action /
|
||||
isolated: it has its own session / participant / run / event / action / lease /
|
||||
approval tables.
|
||||
|
||||
Boundary: this package lives in the harness layer (``deerflow.*``) and never
|
||||
@@ -18,7 +18,8 @@ Tables (see the implementation plan §8):
|
||||
- ``maf_roundtable_runs`` — one execution run of a session
|
||||
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
|
||||
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
|
||||
- ``maf_roundtable_approvals`` — pending human-approval requests
|
||||
- ``maf_roundtable_action_leases`` — active target-write leases
|
||||
- ``maf_roundtable_approvals`` — pending human-approval requests
|
||||
- ``maf_roundtable_seat_runs`` — MAF↔DeerFlow run mapping (survives restart)
|
||||
- ``maf_roundtable_action_secrets`` — encrypted actual execution params (server-only)
|
||||
"""
|
||||
@@ -32,6 +33,7 @@ from deerflow.persistence.maf_roundtable.actions import MafActionRepository
|
||||
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
|
||||
from deerflow.persistence.maf_roundtable.events import MafEventRepository
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
MafRoundtableActionLeaseRow,
|
||||
MafRoundtableActionRow,
|
||||
MafRoundtableActionSecretRow,
|
||||
MafRoundtableApprovalRow,
|
||||
@@ -54,6 +56,7 @@ __all__ = [
|
||||
"MafEventRepository",
|
||||
"MafParticipantRepository",
|
||||
"MafRoundtableActionRow",
|
||||
"MafRoundtableActionLeaseRow",
|
||||
"MafRoundtableActionSecretRow",
|
||||
"MafRoundtableApprovalRow",
|
||||
"MafRoundtableEventRow",
|
||||
|
||||
+14
@@ -10,6 +10,7 @@ store does not re-redact.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -50,6 +51,19 @@ def action_to_dict(row: MafRoundtableActionRow) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def action_to_public_dict(action: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Return the allow-listed action representation exposed by MAF HTTP APIs.
|
||||
|
||||
Target lease keys and idempotency keys are persistence/coordinator
|
||||
details. They must never cross the public boundary.
|
||||
"""
|
||||
|
||||
public = dict(action)
|
||||
public.pop("target_key", None)
|
||||
public.pop("idempotency_key", None)
|
||||
return public
|
||||
|
||||
|
||||
# Back-compat alias for internal callers that used the old private name.
|
||||
_to_dict = action_to_dict
|
||||
|
||||
|
||||
+42
-5
@@ -1,9 +1,9 @@
|
||||
"""ORM models for the MAF (Microsoft Agent Framework) roundtable experiment.
|
||||
|
||||
Six independent tables — **no foreign keys to the legacy ``roundtable_*``
|
||||
Nine independent tables — **no foreign keys to the legacy ``roundtable_*``
|
||||
tables** (``roundtable_jobs`` / ``roundtable_drafts`` /
|
||||
``roundtable_task_drafts``). The MAF experiment is fully isolated: it has its
|
||||
own session / participant / run / event / action / approval tables, keyed by
|
||||
own session / participant / run / event / action / lease / approval tables, keyed by
|
||||
its own surrogate ids, so the legacy multi-agent roundtable keeps its current
|
||||
behavior, schema, and recovery semantics untouched.
|
||||
|
||||
@@ -14,6 +14,7 @@ Tables (see the implementation plan §8):
|
||||
- ``maf_roundtable_runs`` — one execution run of a session
|
||||
- ``maf_roundtable_events`` — monotonic per-run event stream (``(run_id, seq)`` unique)
|
||||
- ``maf_roundtable_actions`` — external write / tool actions (idempotency-keyed)
|
||||
- ``maf_roundtable_action_leases`` — active target-write leases (one per target key)
|
||||
- ``maf_roundtable_approvals`` — pending human-approval requests
|
||||
|
||||
User isolation: ``maf_roundtable_sessions.user_id`` is the owner; all child
|
||||
@@ -263,7 +264,42 @@ class MafRoundtableActionRow(Base):
|
||||
)
|
||||
|
||||
|
||||
# ── 6. approvals ──────────────────────────────────────────────────────────
|
||||
# ── 6. target leases ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MafRoundtableActionLeaseRow(Base):
|
||||
"""One active, short-lived lease for a server-derived external target key.
|
||||
|
||||
The key is opaque (a stable SHA-256 digest) and is never sourced from a
|
||||
browser/model field. ``target_key`` is the primary key, which makes the
|
||||
database — not an in-process mutex — arbitrate concurrent writes across
|
||||
workers and restarts. A lease belongs to exactly one action and is removed
|
||||
with a known terminal result; an ``unknown`` result deliberately retains a
|
||||
manual-reconciliation barrier.
|
||||
"""
|
||||
|
||||
__tablename__ = "maf_roundtable_action_leases"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("action_id", name="uq_maf_action_leases_action"),
|
||||
)
|
||||
|
||||
target_key: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
action_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("maf_roundtable_actions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
lease_expires_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
|
||||
)
|
||||
|
||||
|
||||
# ── 7. approvals ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MafRoundtableApprovalRow(Base):
|
||||
@@ -297,7 +333,7 @@ class MafRoundtableApprovalRow(Base):
|
||||
)
|
||||
|
||||
|
||||
# ── 7. seat runs (Phase 2: MAF↔DeerFlow run mapping) ─────────────────────
|
||||
# ── 8. seat runs (Phase 2: MAF↔DeerFlow run mapping) ─────────────────────
|
||||
|
||||
|
||||
class MafRoundtableSeatRunRow(Base):
|
||||
@@ -370,7 +406,7 @@ class MafRoundtableSeatRunRow(Base):
|
||||
)
|
||||
|
||||
|
||||
# ── 8. action secrets (encrypted actual execution params, server-only) ────
|
||||
# ── 9. action secrets (encrypted actual execution params, server-only) ────
|
||||
|
||||
|
||||
class MafRoundtableActionSecretRow(Base):
|
||||
@@ -403,6 +439,7 @@ class MafRoundtableActionSecretRow(Base):
|
||||
|
||||
__all__ = [
|
||||
"MafRoundtableActionRow",
|
||||
"MafRoundtableActionLeaseRow",
|
||||
"MafRoundtableActionSecretRow",
|
||||
"MafRoundtableApprovalRow",
|
||||
"MafRoundtableEventRow",
|
||||
|
||||
+310
-5
@@ -14,19 +14,24 @@ another's transactions.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.maf_roundtable.action_secrets import MafActionSecretRepository
|
||||
from deerflow.persistence.maf_roundtable.actions import MafActionRepository, action_to_dict
|
||||
from deerflow.persistence.maf_roundtable.actions import (
|
||||
MafActionRepository,
|
||||
MafActionStateMachineError,
|
||||
action_to_dict,
|
||||
)
|
||||
from deerflow.persistence.maf_roundtable.approvals import MafApprovalRepository
|
||||
from deerflow.persistence.maf_roundtable.events import MafEventRepository
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
MafRoundtableActionLeaseRow,
|
||||
MafRoundtableActionRow,
|
||||
MafRoundtableActionSecretRow,
|
||||
MafRoundtableApprovalRow,
|
||||
@@ -315,6 +320,7 @@ class MafRoundtableStore:
|
||||
approval_payload: dict[str, Any] | None,
|
||||
audit_event_agent_id: str,
|
||||
audit_event_turn_index: int,
|
||||
approval_expires_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create an action + secret + approval + audit event in ONE transaction.
|
||||
|
||||
@@ -432,6 +438,7 @@ class MafRoundtableStore:
|
||||
status="pending",
|
||||
summary=approval_summary,
|
||||
payload_redacted_json=approval_payload,
|
||||
expires_at=approval_expires_at,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
))
|
||||
@@ -482,10 +489,308 @@ class MafRoundtableStore:
|
||||
return result
|
||||
|
||||
raise RuntimeError(
|
||||
f"create_pending_action_bundle failed after {_MAX_BUNDLE_RETRIES} attempts "
|
||||
f"for idempotency_key={idempotency_key}"
|
||||
f"create_pending_action_bundle failed after {_MAX_BUNDLE_RETRIES} attempts"
|
||||
) from last_error
|
||||
|
||||
async def claim_action_for_execution(self, action_id: str, *, lease_seconds: int) -> str:
|
||||
"""Atomically claim an approved action and its external-target lease.
|
||||
|
||||
The action CAS and the target lease live in one transaction. A second
|
||||
approved action for the same target receives ``target_busy`` and is not
|
||||
executed. An expired lease owned by a still-running action is treated
|
||||
as an ambiguous external outcome: the old action becomes ``unknown``
|
||||
and the caller receives ``target_reconciliation_required`` rather than
|
||||
being allowed to write over it.
|
||||
"""
|
||||
# Pre-Phase-5 actions have no server-derived target key. Preserve the
|
||||
# original conditional UPDATE CAS for them; replacing it with a
|
||||
# read-then-write transaction would weaken their existing cross-worker
|
||||
# at-most-once guarantee on SQLite (where FOR UPDATE is a no-op).
|
||||
existing = await self.actions.get_action(action_id)
|
||||
if existing is None:
|
||||
raise MafActionStateMachineError(f"action {action_id} not found during claim")
|
||||
if not existing.get("target_key"):
|
||||
return await self.actions.claim_for_execution(action_id)
|
||||
|
||||
now = _now()
|
||||
expiry = now + timedelta(seconds=max(1, lease_seconds))
|
||||
last_error: Exception | None = None
|
||||
|
||||
for _attempt in range(4):
|
||||
try:
|
||||
async with self._sf() as db:
|
||||
async with db.begin():
|
||||
action = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionRow)
|
||||
.where(MafRoundtableActionRow.id == action_id)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalars().first()
|
||||
if action is None:
|
||||
raise MafActionStateMachineError(
|
||||
f"action {action_id} not found during claim"
|
||||
)
|
||||
if action.status == "running":
|
||||
return "in_progress"
|
||||
if action.status in ("succeeded", "failed", "unknown", "rejected"):
|
||||
return str(action.status)
|
||||
if action.status != "approved":
|
||||
raise MafActionStateMachineError(
|
||||
f"action {action_id} cannot be claimed from state '{action.status}'"
|
||||
)
|
||||
|
||||
# Legacy rows can be executed at most once with the old
|
||||
# action CAS. All new actions receive a target key from
|
||||
# ActionCoordinator; never manufacture one from client
|
||||
# input here.
|
||||
if action.target_key:
|
||||
lease = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionLeaseRow)
|
||||
.where(MafRoundtableActionLeaseRow.target_key == action.target_key)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalars().first()
|
||||
if lease is not None and lease.action_id != action.id:
|
||||
owner = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionRow)
|
||||
.where(MafRoundtableActionRow.id == lease.action_id)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalars().first()
|
||||
if owner is not None and owner.status == "unknown":
|
||||
# Unknown means the remote side effect may
|
||||
# have happened. Keep the lease as a
|
||||
# durable manual-reconciliation barrier;
|
||||
# another write to the same object must not
|
||||
# silently leapfrog it.
|
||||
return "target_reconciliation_required"
|
||||
if lease.lease_expires_at > now:
|
||||
return "target_busy"
|
||||
if owner is not None and owner.status == "running":
|
||||
owner.status = "unknown"
|
||||
owner.result_redacted_json = {
|
||||
"error_code": "target_lease_expired_during_execution"
|
||||
}
|
||||
owner.finished_at = now
|
||||
owner.updated_at = now
|
||||
# Retain the lease as an explicit manual
|
||||
# reconciliation barrier. It is released
|
||||
# only by a future audited reconciliation
|
||||
# flow, never by automatic replay.
|
||||
lease.lease_expires_at = now
|
||||
lease.updated_at = now
|
||||
return "target_reconciliation_required"
|
||||
await db.delete(lease)
|
||||
elif lease is not None and lease.action_id == action.id:
|
||||
# This can only occur after a failed local
|
||||
# terminal-result write. Do not treat it as a
|
||||
# new execution opportunity.
|
||||
return "in_progress"
|
||||
|
||||
db.add(MafRoundtableActionLeaseRow(
|
||||
target_key=action.target_key,
|
||||
action_id=action.id,
|
||||
lease_expires_at=expiry,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
))
|
||||
|
||||
action.status = "running"
|
||||
action.attempt_count = int(action.attempt_count or 0) + 1
|
||||
action.started_at = now
|
||||
action.updated_at = now
|
||||
await db.flush()
|
||||
return "claimed"
|
||||
except IntegrityError as exc:
|
||||
# The target-key primary key is the cross-worker arbitration
|
||||
# point. On a collision retry and observe the winning lease.
|
||||
last_error = exc
|
||||
continue
|
||||
|
||||
raise RuntimeError(f"could not claim action lease for {action_id}") from last_error
|
||||
|
||||
async def record_action_terminal_result(
|
||||
self,
|
||||
action_id: str,
|
||||
*,
|
||||
status: str,
|
||||
result: dict[str, Any] | None = None,
|
||||
external_object_id: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Persist a terminal result and release a known target lease atomically.
|
||||
|
||||
``unknown`` is intentionally different: its target lease remains as a
|
||||
manual-reconciliation barrier because the remote side effect may have
|
||||
completed even though the local process cannot prove it.
|
||||
"""
|
||||
if status not in ("succeeded", "failed", "unknown"):
|
||||
raise ValueError(f"invalid action result status: {status}")
|
||||
now = _now()
|
||||
async with self._sf() as db:
|
||||
async with db.begin():
|
||||
action = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionRow)
|
||||
.where(MafRoundtableActionRow.id == action_id)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalars().first()
|
||||
if action is None or action.status != "running":
|
||||
return None
|
||||
action.status = status
|
||||
action.result_redacted_json = result
|
||||
action.external_object_id = external_object_id
|
||||
action.finished_at = now
|
||||
action.updated_at = now
|
||||
if status != "unknown":
|
||||
# A known success or known failure has a deterministic
|
||||
# outcome, so the next approved action may acquire the
|
||||
# target. ``unknown`` deliberately retains the barrier.
|
||||
await db.execute(
|
||||
delete(MafRoundtableActionLeaseRow).where(
|
||||
MafRoundtableActionLeaseRow.action_id == action_id
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
return action_to_dict(action)
|
||||
|
||||
async def expire_due_approvals(
|
||||
self, *, run_id: str | None = None, now: datetime | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Expire due approvals and reject unexecuted actions with audit events.
|
||||
|
||||
Expiry is deliberately a fail-closed transition: only an action still
|
||||
waiting for approval becomes rejected; a later/running action is never
|
||||
overwritten. The approval, action and critical event are committed in
|
||||
one transaction so polling sees a coherent terminal decision.
|
||||
"""
|
||||
current = now or _now()
|
||||
last_error: Exception | None = None
|
||||
for _attempt in range(4):
|
||||
try:
|
||||
async with self._sf() as db:
|
||||
async with db.begin():
|
||||
stmt = select(MafRoundtableApprovalRow).where(
|
||||
MafRoundtableApprovalRow.status == "pending",
|
||||
MafRoundtableApprovalRow.expires_at.is_not(None),
|
||||
MafRoundtableApprovalRow.expires_at <= current,
|
||||
)
|
||||
if run_id is not None:
|
||||
stmt = stmt.where(MafRoundtableApprovalRow.run_id == run_id)
|
||||
approvals = (await db.execute(stmt.with_for_update())).scalars().all()
|
||||
if not approvals:
|
||||
return []
|
||||
|
||||
max_seq_by_run: dict[str, int] = {}
|
||||
expired: list[dict[str, Any]] = []
|
||||
for approval in approvals:
|
||||
approval.status = "expired"
|
||||
approval.decided_by = "system:approval_timeout"
|
||||
approval.decided_at = current
|
||||
approval.updated_at = current
|
||||
action = None
|
||||
if approval.action_id:
|
||||
action = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionRow)
|
||||
.where(MafRoundtableActionRow.id == approval.action_id)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalars().first()
|
||||
if action is not None and action.status == "waiting_approval":
|
||||
action.status = "rejected"
|
||||
action.finished_at = current
|
||||
action.updated_at = current
|
||||
|
||||
if approval.run_id and action is not None:
|
||||
seq = max_seq_by_run.get(approval.run_id)
|
||||
if seq is None:
|
||||
seq = int((
|
||||
await db.execute(
|
||||
select(func.max(MafRoundtableEventRow.seq)).where(
|
||||
MafRoundtableEventRow.run_id == approval.run_id
|
||||
)
|
||||
)
|
||||
).scalar_one() or 0)
|
||||
seq += 1
|
||||
max_seq_by_run[approval.run_id] = seq
|
||||
db.add(MafRoundtableEventRow(
|
||||
id=uuid4().hex,
|
||||
session_id=approval.session_id,
|
||||
run_id=approval.run_id,
|
||||
seq=seq,
|
||||
event_type="action.rejected",
|
||||
agent_id=approval.requested_by_agent_id,
|
||||
turn_index=action.turn_index,
|
||||
source_event_id=f"approval-expired:{approval.id}",
|
||||
payload_json={
|
||||
"action_id": action.id,
|
||||
"approval_id": approval.id,
|
||||
"reason": "approval_expired",
|
||||
},
|
||||
created_at=current,
|
||||
))
|
||||
expired.append({
|
||||
"id": approval.id,
|
||||
"run_id": approval.run_id,
|
||||
"action_id": approval.action_id,
|
||||
})
|
||||
await db.flush()
|
||||
return expired
|
||||
except IntegrityError as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
raise RuntimeError("could not persist expired MAF approvals") from last_error
|
||||
|
||||
async def recover_expired_action_leases(
|
||||
self, *, run_id: str, now: datetime | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Mark stale in-flight writes ``unknown`` during restart recovery.
|
||||
|
||||
No tool is re-run. A lease that survived a process death cannot prove
|
||||
whether the remote side effect completed, so the only safe recovery is
|
||||
an explicit human reconciliation path.
|
||||
"""
|
||||
current = now or _now()
|
||||
async with self._sf() as db:
|
||||
async with db.begin():
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(MafRoundtableActionLeaseRow, MafRoundtableActionRow)
|
||||
.join(
|
||||
MafRoundtableActionRow,
|
||||
MafRoundtableActionLeaseRow.action_id == MafRoundtableActionRow.id,
|
||||
)
|
||||
.where(
|
||||
MafRoundtableActionRow.run_id == run_id,
|
||||
MafRoundtableActionLeaseRow.lease_expires_at <= current,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
).all()
|
||||
recovered: list[dict[str, Any]] = []
|
||||
for lease, action in rows:
|
||||
if action.status == "running":
|
||||
action.status = "unknown"
|
||||
action.result_redacted_json = {
|
||||
"error_code": "target_lease_expired_during_execution"
|
||||
}
|
||||
action.finished_at = current
|
||||
action.updated_at = current
|
||||
recovered.append(action_to_dict(action))
|
||||
# Keep a durable block for manual reconciliation;
|
||||
# automatically deleting it would allow a later write
|
||||
# to overtake an ambiguous external side effect.
|
||||
lease.lease_expires_at = current
|
||||
lease.updated_at = current
|
||||
elif action.status != "unknown":
|
||||
await db.delete(lease)
|
||||
await db.flush()
|
||||
return recovered
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MafActionRepository",
|
||||
|
||||
Reference in new issue
Block a user