feat(maf): complete resilient roundtable workflow

This commit is contained in:
董宏凯 committed 2026-08-02 00:23:25 +08:00
1 parent bd4131b4e2
commit 21c4c898e7
23 files changed
+4810 -312

No files matched your search

@@ -23,12 +23,15 @@ from typing import Any
from app.maf_roundtable.action_coordinator import ActionCoordinator
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
from app.maf_roundtable.fact_ledger import record_action_transition
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
from app.maf_roundtable.sanitize import sanitize
from app.maf_roundtable.workflow_executors import (
ApprovalGateExecutor,
CoordinatorExecutor,
CrossReviewExecutor,
FinalizerExecutor,
PlannerExecutor,
RoundtableState,
SeatExecutor,
WaitingTerminalExecutor,
@@ -92,6 +95,7 @@ class WorkflowRunner:
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
manager_id = session.get("manager_agent_id")
seat_timeout, schema_repair = self._orchestration_settings()
state = RoundtableState(
session_id=session_id,
@@ -106,28 +110,11 @@ class WorkflowRunner:
seat_agent_ids=seat_agents,
thread_map=thread_map,
capability_snapshot=session.get("capability_snapshot"),
seat_timeout_seconds=seat_timeout,
schema_repair_enabled=schema_repair,
)
# Build executor instances — one per seat, each bound to a fixed agent_id.
coordinator = CoordinatorExecutor(self._adapter, state)
seat_executors = [
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
for agent_id in seat_agents
]
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
finalizer = FinalizerExecutor(self._adapter, state)
waiting_terminal = WaitingTerminalExecutor()
workflow = create_workflow(
mode=state.mode,
coordinator=coordinator,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=seat_agents,
)
workflow = self._build_workflow(state)
await self._run_workflow(workflow, state)
# Persist final state. waiting_approval is NOT a terminal state —
@@ -162,35 +149,103 @@ class WorkflowRunner:
except Exception as exc:
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
def _orchestration_settings(self) -> tuple[float, bool]:
try:
from deerflow.config import get_app_config
cfg = get_app_config().maf_roundtable
return (
float(getattr(cfg, "seat_timeout_seconds", 600) or 600),
bool(getattr(cfg, "schema_repair_enabled", True)),
)
except Exception: # noqa: BLE001
return 600.0, True
def _build_workflow(self, state: RoundtableState) -> Any:
seat_executors = [
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
for agent_id in state.seat_agent_ids
]
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
finalizer = FinalizerExecutor(self._adapter, state)
waiting_terminal = WaitingTerminalExecutor()
if state.mode == "group_chat":
coordinator = CoordinatorExecutor(self._adapter, state)
return create_workflow(
mode=state.mode,
coordinator=coordinator,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=state.seat_agent_ids,
)
planner = PlannerExecutor(self._adapter, state)
cross_review = CrossReviewExecutor(self._adapter, state)
return create_workflow(
mode="structured",
planner=planner,
cross_review=cross_review,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=state.seat_agent_ids,
)
async def _persist_checkpoint(self, state: RoundtableState, *, reason: str = "") -> None:
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
await self._adapter._emit_critical(
state.run_id,
make_workflow_checkpointed(
None,
state.current_round,
checkpoint_id=f"nodes-{len(state.completed_nodes)}",
reason=reason or f"stage={state.stage}",
),
)
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
"""Execute the workflow graph via the MAF SDK's streaming API.
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
events. Persists the RoundtableState checkpoint after each superstep.
"""
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
event_type = getattr(event, "type", "")
executor_id = getattr(event, "executor_id", None)
if event_type == "superstep_completed":
# Persist checkpoint after each superstep.
iteration = getattr(event, "iteration", None)
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
executor_id, state.current_round,
checkpoint_id=f"superstep-{iteration}",
reason="superstep_completed",
))
await self._persist_checkpoint(
state, reason=f"superstep_completed:{iteration}"
)
elif event_type in ("executor_completed", "executor_invoked"):
# Persist after each node so completed_nodes survive restarts.
if state.completed_nodes:
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
elif event_type == "executor_failed":
details = getattr(event, "details", None)
error_msg = str(details) if details else "executor failed"
# Soft seat failures are handled inside SeatExecutor; only hard
# failures (e.g. WriteGuardConfigurationError) abort the run.
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
# Always persist latest completed_nodes at end of stream.
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
# Check if the workflow ended in a waiting_approval state.
if state.status == "waiting_approval" or state.pending_approvals:
state.status = "waiting_approval"
@@ -286,10 +341,22 @@ class WorkflowRunner:
for pa in state.pending_approvals:
approval_id = pa.get("approval_id", "")
action_id = pa.get("action_id", "")
# Extracted up-front so EVERY branch (rejected / failed / approved)
# can write the ledger transition with the correct agent_id.
agent_id = pa.get("agent_id", "")
thread_id = state.thread_map.get(agent_id, "")
tool_name = pa.get("tool_name", "")
approval = await self._service.store.approvals.get_approval(approval_id)
if approval is None:
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": "approval not found"})
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="failed",
agent_id=agent_id,
)
continue
approval_status = approval.get("status", "pending")
@@ -307,17 +374,32 @@ class WorkflowRunner:
decided_by=approval.get("decided_by", ""),
))
action_outcomes.append({"action_id": action_id, "status": "rejected"})
# Phase-3 P1-3: move the ledger row out of pending into the
# failed/unknown partition as rejected.
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="rejected",
agent_id=agent_id,
)
continue
if approval_status != "approved":
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": f"unexpected approval status: {approval_status}"})
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="failed",
agent_id=agent_id,
)
continue
# Approved — execute deterministically via ToolExecutor.
agent_id = pa.get("agent_id", "")
thread_id = state.thread_map.get(agent_id, "")
tool_name = pa.get("tool_name", "")
# (agent_id / thread_id / tool_name are already extracted at the
# top of this loop iteration so every branch can use them.)
# Conditional transition (waiting_approval → approved). The
# authoritative at-most-once gate is the ToolExecutor's
@@ -378,11 +460,20 @@ class WorkflowRunner:
"error": outcome.get("error"),
})
state.fact_ledger["pending_actions"].append({
"action_id": action_id,
"tool_name": tool_name,
"outcome": outcome.get("status"),
})
out_status = outcome.get("status")
ledger_status = {
"succeeded": "executed",
"failed": "failed",
"unknown": "unknown",
"skipped": "executed",
}.get(str(out_status), "unknown")
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=tool_name,
status=ledger_status,
agent_id=agent_id,
)
# P0-3: determine parent run status from action outcomes.
statuses = {o.get("status") for o in action_outcomes}
@@ -451,6 +542,76 @@ class WorkflowRunner:
return await self._service.store.runs.get_run(run_id)
async def recover(self, run_id: str) -> dict[str, Any] | None:
"""Graph-level recovery after process restart (MAF-03-07).
Reloads ``RoundtableState`` from ``maf_checkpoint`` and re-enters the
structured workflow. Executors whose ids are in ``completed_nodes``
short-circuit without calling DeerFlow again.
"""
t0 = time.perf_counter()
try:
import agent_framework # noqa: F401
except ImportError as exc:
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
run = await self._service.store.runs.get_run(run_id)
if run is None:
return None
if run["status"] not in ("running", "waiting_approval"):
logger.info("recover(%s): status=%s — nothing to recover", run_id, run["status"])
return run
checkpoint = run.get("maf_checkpoint")
if not checkpoint:
logger.warning("recover(%s): no checkpoint", run_id)
return run
state = RoundtableState.from_dict(checkpoint)
session = await self._service.store.sessions.get_session(state.session_id, user_id=None)
if session is None:
return run
# Ensure threads still bound.
participants = await self._service.store.participants.list_participants(state.session_id)
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
state.thread_map.update(thread_map)
if run["status"] == "waiting_approval":
# Approval path stays on the existing resume() — recover only
# continues orchestration for running graphs.
return run
await self._service.store.runs.update_run(run_id, {"status": "running"})
try:
workflow = self._build_workflow(state)
await self._run_workflow(workflow, state)
latency = int((time.perf_counter() - t0) * 1000)
final_status = state.status
update_fields: dict[str, Any] = {
"status": final_status,
"latency_ms": latency,
"maf_checkpoint": state.to_dict(),
"token_usage": _aggregate_token_usage(state.turn_results),
}
if final_status in ("completed", "failed", "cancelled"):
update_fields["finished_at"] = datetime.now(UTC)
await self._service.store.runs.update_run(run_id, update_fields)
session_fields: dict[str, Any] = {"status": final_status}
if final_status == "completed":
session_fields["final_result"] = state.final_result
if final_status != "waiting_approval":
session_fields["fact_ledger"] = state.fact_ledger
await self._service.store.sessions.update_session(
state.session_id, user_id=self._user_id, fields=session_fields
)
except Exception as exc: # noqa: BLE001
await self._fail_run(run_id, state.session_id, t0, type(exc).__name__, str(exc))
return await self._service.store.runs.get_run(run_id)
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
"""Aggregate token usage across all seat turns into a run-level summary."""