feat(maf): add secure roundtable workflow runtime

This commit is contained in:
董宏凯 committed 2026-08-01 17:51:53 +08:00
1 parent e690475e6e
commit bd4131b4e2
52 files changed
+10315 -96

No files matched your search

@@ -0,0 +1,483 @@
"""MAF Workflow runner — orchestrates workflow execution for a MAF run.
Called after ``create_run`` to actually execute the workflow. Translates all
MAF Workflow events into ``maf_roundtable_events`` (the single event stream
the frontend reads). Manages run status lifecycle and checkpoint persistence.
There is NO sequential fallback. If the MAF SDK is missing or the workflow
fails, the run is marked ``failed`` with a sanitized error message. The runner
never pretends a workflow executed when it did not.
Usage::
runner = WorkflowRunner(app, service=svc, user_id="u1")
await runner.execute(session, run)
"""
from __future__ import annotations
import logging
import time
from datetime import UTC, datetime
from typing import Any
from app.maf_roundtable.action_coordinator import ActionCoordinator
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
from app.maf_roundtable.sanitize import sanitize
from app.maf_roundtable.workflow_executors import (
ApprovalGateExecutor,
CoordinatorExecutor,
FinalizerExecutor,
RoundtableState,
SeatExecutor,
WaitingTerminalExecutor,
_agent_capabilities_from_snapshot,
generate_final_report,
)
from app.maf_roundtable.workflow_factory import create_workflow
logger = logging.getLogger(__name__)
class MafSdkMissingError(RuntimeError):
"""Raised when the MAF SDK (agent_framework) is not installed."""
class WorkflowRunner:
"""Executes a MAF workflow for a given session/run pair."""
def __init__(self, app: Any, *, service: Any, user_id: str) -> None:
self._app = app
self._service = service
self._user_id = user_id
self._client = DeerFlowRuntimeClient(app)
self._adapter = DeerFlowAgentAdapter(self._client, service=service, user_id=user_id)
self._action_coordinator = ActionCoordinator(service, user_id=user_id)
@property
def adapter(self) -> DeerFlowAgentAdapter:
return self._adapter
@property
def action_coordinator(self) -> ActionCoordinator:
return self._action_coordinator
async def execute(self, session: dict[str, Any], run: dict[str, Any]) -> None:
"""Execute the full workflow for a session/run.
Transitions run status: pending → running → completed/failed/cancelled.
Persists checkpoint (RoundtableState) after each major step so the
workflow can resume after approval or restart.
"""
run_id = run["id"]
session_id = session["id"]
t0 = time.perf_counter()
# Verify the MAF SDK is available before touching anything.
try:
import agent_framework # noqa: F401
except ImportError as exc:
await self._fail_run(run_id, session_id, t0, "MafSdkMissing", "agent_framework SDK is not installed")
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
await self._service.store.runs.update_run(run_id, {
"status": "running",
"started_at": datetime.now(UTC),
})
try:
participants = await self._service.store.participants.list_participants(session_id)
thread_map = await self._adapter.ensure_threads(session_id, run_id, participants)
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
manager_id = session.get("manager_agent_id")
state = RoundtableState(
session_id=session_id,
run_id=run_id,
task_text=session.get("task_text", ""),
mode=session.get("mode", "structured"),
model_name=session.get("model_name"),
reasoning_effort=session.get("reasoning_effort"),
max_rounds=session.get("max_rounds", 8),
write_policy=session.get("write_policy", "confirm_high_risk"),
manager_agent_id=manager_id,
seat_agent_ids=seat_agents,
thread_map=thread_map,
capability_snapshot=session.get("capability_snapshot"),
)
# Build executor instances — one per seat, each bound to a fixed agent_id.
coordinator = CoordinatorExecutor(self._adapter, state)
seat_executors = [
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
for agent_id in seat_agents
]
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
finalizer = FinalizerExecutor(self._adapter, state)
waiting_terminal = WaitingTerminalExecutor()
workflow = create_workflow(
mode=state.mode,
coordinator=coordinator,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=seat_agents,
)
await self._run_workflow(workflow, state)
# Persist final state. waiting_approval is NOT a terminal state —
# it must NOT write finished_at. Only completed/failed/cancelled do.
latency = int((time.perf_counter() - t0) * 1000)
final_status = state.status
token_usage = _aggregate_token_usage(state.turn_results)
update_fields: dict[str, Any] = {
"status": final_status,
"latency_ms": latency,
"maf_checkpoint": state.to_dict(),
"token_usage": token_usage,
}
# Only write finished_at for terminal statuses.
if final_status in ("completed", "failed", "cancelled"):
update_fields["finished_at"] = datetime.now(UTC)
await self._service.store.runs.update_run(run_id, update_fields)
session_fields: dict[str, Any] = {"status": final_status}
if final_status == "completed":
session_fields["final_result"] = state.final_result
if final_status != "waiting_approval":
session_fields["fact_ledger"] = state.fact_ledger
await self._service.store.sessions.update_session(
session_id, user_id=self._user_id, fields=session_fields
)
except MafSdkMissingError:
raise
except Exception as exc:
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
"""Execute the workflow graph via the MAF SDK's streaming API.
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
events. Persists the RoundtableState checkpoint after each superstep.
"""
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
event_type = getattr(event, "type", "")
executor_id = getattr(event, "executor_id", None)
if event_type == "superstep_completed":
# Persist checkpoint after each superstep.
iteration = getattr(event, "iteration", None)
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
executor_id, state.current_round,
checkpoint_id=f"superstep-{iteration}",
reason="superstep_completed",
))
elif event_type == "executor_failed":
details = getattr(event, "details", None)
error_msg = str(details) if details else "executor failed"
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
# Check if the workflow ended in a waiting_approval state.
if state.status == "waiting_approval" or state.pending_approvals:
state.status = "waiting_approval"
async def _fail_run(self, run_id: str, session_id: str, t0: float, error_code: str, error_message: str) -> None:
"""Mark a run as failed with a sanitized error message."""
latency = int((time.perf_counter() - t0) * 1000)
safe_message = str(sanitize(error_message))[:500]
logger.error("MAF workflow failed for run %s: %s", run_id, safe_message, exc_info=True)
await self._service.store.runs.update_run(run_id, {
"status": "failed",
"finished_at": datetime.now(UTC),
"error_code": error_code,
"error_message_safe": safe_message,
"latency_ms": latency,
})
await self._service.store.sessions.update_session(session_id, user_id=self._user_id, fields={
"status": "failed",
})
async def cancel(self, run_id: str) -> dict[str, Any]:
"""Cancel a running workflow (cascades to all active seat runs).
Returns the structured cancel result from the adapter so the caller
can decide whether to finalize the parent run as ``cancelled`` or
leave it non-terminal when some sub-runs failed to cancel.
"""
return await self._adapter.cancel(run_id)
async def resume(self, run_id: str) -> dict[str, Any] | None:
"""Resume a workflow from its persisted checkpoint after an approval decision.
P1: ``RoundtableState.from_dict()`` is actually called here. Flow:
1. Load ``maf_checkpoint_json`` from the run row.
2. Restore ``RoundtableState`` via ``from_dict()``.
3. Check the pending approvals' status:
- ``approved``: execute the approved action via ToolExecutor
(at-most-once, deterministic, no LLM).
- ``rejected`` / ``expired``: write ``action.rejected``, skip.
- ``pending``: do NOT start a second Workflow — return immediately.
4. After all actions are executed, determine the parent run status:
- All succeeded → ``completed``.
- Any failed → ``failed`` (P1-8).
- Any unknown → ``waiting_input`` (not completed; human reconcile needed).
- All skipped (already executed) → ``completed``.
Only the owner can resume (enforced by the router before calling
``dispatcher.resume``).
"""
t0 = time.perf_counter()
run = await self._service.store.runs.get_run(run_id)
if run is None:
return None
if run["status"] != "waiting_approval":
return run
checkpoint = run.get("maf_checkpoint")
if not checkpoint:
logger.warning("Cannot resume run %s: no checkpoint", run_id)
return run
state = RoundtableState.from_dict(checkpoint)
session_id = state.session_id
if not state.pending_approvals:
logger.warning("Cannot resume run %s: no pending_approvals in checkpoint", run_id)
return run
# Check if any approval is still pending — if so, don't resume yet.
any_pending = False
for pa in state.pending_approvals:
approval_id = pa.get("approval_id", "")
approval = await self._service.store.approvals.get_approval(approval_id)
if approval and approval.get("status") == "pending":
any_pending = True
if any_pending:
return run
# P0-3 run-level CAS: only ONE concurrent resume may execute the
# approved actions and write the parent run's final state. The winner
# moves the run waiting_approval → running; every other concurrent
# caller loses the CAS and returns the current run WITHOUT writing a
# terminal status (so it can never race a completed/failed write).
claimed_run = await self._service.store.runs.claim_for_resume(run_id)
if not claimed_run:
logger.info("resume(%s): lost run-level CAS — another resume is executing", run_id)
return await self._service.store.runs.get_run(run_id)
# Execute each approved action (skip rejected/expired).
action_outcomes: list[dict[str, Any]] = []
for pa in state.pending_approvals:
approval_id = pa.get("approval_id", "")
action_id = pa.get("action_id", "")
approval = await self._service.store.approvals.get_approval(approval_id)
if approval is None:
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": "approval not found"})
continue
approval_status = approval.get("status", "pending")
if approval_status in ("rejected", "expired"):
from app.maf_roundtable.event_normalizer import make_action_rejected
# Conditional transition (waiting_approval → rejected) — never
# overwrite a terminal state via a generic update.
await self._service.store.actions.mark_rejected(action_id)
await self._adapter._emit_critical(run_id, make_action_rejected(
pa.get("agent_id", ""),
state.current_round,
action_id=action_id,
approval_id=approval_id,
decided_by=approval.get("decided_by", ""),
))
action_outcomes.append({"action_id": action_id, "status": "rejected"})
continue
if approval_status != "approved":
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": f"unexpected approval status: {approval_status}"})
continue
# Approved — execute deterministically via ToolExecutor.
agent_id = pa.get("agent_id", "")
thread_id = state.thread_map.get(agent_id, "")
tool_name = pa.get("tool_name", "")
# Conditional transition (waiting_approval → approved). The
# authoritative at-most-once gate is the ToolExecutor's
# approved → running claim below; this just advances the state
# machine without ever clobbering a terminal status.
await self._service.store.actions.mark_approved(action_id)
from app.maf_roundtable.event_normalizer import make_action_approved
await self._adapter._emit_critical(run_id, make_action_approved(
agent_id,
state.current_round,
action_id=action_id,
approval_id=approval_id,
decided_by=approval.get("decided_by", ""),
))
# Params are recovered server-side inside ToolExecutor (after the
# atomic occupy). A missing/undecryptable secret fails the action
# there — it is never coerced into empty params, so the tool is
# never called (execution count == 0) and the parent run fails.
if thread_id and tool_name:
from app.maf_roundtable.tool_executor import ToolExecutor
executor = ToolExecutor(
adapter=self._adapter,
action_coordinator=self._action_coordinator,
user_id=self._user_id,
)
try:
outcome = await executor.execute(
action_id=action_id,
session_id=session_id,
run_id=run_id,
agent_id=agent_id,
thread_id=thread_id,
approved_tool=tool_name,
capability_snapshot=state.capability_snapshot,
write_policy=state.write_policy,
model_name=state.model_name,
excluded_tools=list(
_agent_capabilities_from_snapshot(
state.capability_snapshot, agent_id
).get("excluded_tools")
or []
),
)
except Exception as exc:
safe_msg = str(sanitize(str(exc)))[:500]
logger.error("ToolExecutor for action %s failed: %s", action_id, safe_msg)
outcome = {"status": "failed", "error": safe_msg}
else:
outcome = {"status": "failed", "error": f"no thread_id or tool_name for action {action_id}"}
action_outcomes.append({
"action_id": action_id,
"tool_name": tool_name,
"status": outcome.get("status"),
"error": outcome.get("error"),
})
state.fact_ledger["pending_actions"].append({
"action_id": action_id,
"tool_name": tool_name,
"outcome": outcome.get("status"),
})
# P0-3: determine parent run status from action outcomes.
statuses = {o.get("status") for o in action_outcomes}
# While any action is still in_progress (another executor running it),
# the parent run stays running — never write finished_at, never finalize.
if "in_progress" in statuses:
state.status = "running"
await self._service.store.runs.update_run(run_id, {
"status": "running",
"maf_checkpoint": state.to_dict(),
})
logger.info("resume(%s): action(s) still in_progress — run stays running", run_id)
return await self._service.store.runs.get_run(run_id)
state.pending_approvals = []
has_unknown = "unknown" in statuses
has_failed = "failed" in statuses
all_resolved = statuses <= {"succeeded", "skipped", "rejected"}
if has_unknown:
state.status = "waiting_input" # human reconciliation needed
elif has_failed:
state.status = "failed"
elif all_resolved:
# Re-enter the MAF finalizer path to synthesize the real report —
# do NOT just flip the status to completed.
try:
await generate_final_report(self._adapter, state)
state.status = "completed"
except Exception as exc: # noqa: BLE001
safe_msg = str(sanitize(str(exc)))[:500]
logger.error("resume(%s): finalizer failed: %s", run_id, safe_msg)
state.status = "failed"
action_outcomes.append({"action_id": "", "status": "failed",
"error": f"finalizer failed: {safe_msg}"})
else:
state.status = "failed"
# Persist final state.
latency = int((time.perf_counter() - t0) * 1000)
token_usage = _aggregate_token_usage(state.turn_results)
update_fields: dict[str, Any] = {
"status": state.status,
"latency_ms": latency,
"maf_checkpoint": state.to_dict(),
"token_usage": token_usage,
}
if state.status in ("completed", "failed", "cancelled"):
update_fields["finished_at"] = datetime.now(UTC)
if state.status == "failed":
errors = [o.get("error") for o in action_outcomes if o.get("error")]
update_fields["error_code"] = "ActionExecutionFailed"
update_fields["error_message_safe"] = str(sanitize("; ".join(errors)))[:500]
await self._service.store.runs.update_run(run_id, update_fields)
session_fields: dict[str, Any] = {"status": state.status}
if state.status == "completed":
session_fields["final_result"] = state.final_result
session_fields["fact_ledger"] = state.fact_ledger
await self._service.store.sessions.update_session(
session_id, user_id=self._user_id, fields=session_fields
)
return await self._service.store.runs.get_run(run_id)
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
"""Aggregate token usage across all seat turns into a run-level summary."""
total_input = 0
total_output = 0
total_total = 0
per_seat: list[dict[str, Any]] = []
for turn in turn_results:
usage = turn.get("usage")
if not isinstance(usage, dict):
continue
inp = usage.get("input_tokens", 0) or usage.get("prompt_tokens", 0) or 0
out = usage.get("output_tokens", 0) or usage.get("completion_tokens", 0) or 0
tot = usage.get("total_tokens", 0) or (inp + out)
total_input += inp
total_output += out
total_total += tot
per_seat.append({
"agent_id": turn.get("agent_id", ""),
"turn_index": turn.get("turn_index", 0),
"input_tokens": inp,
"output_tokens": out,
"total_tokens": tot,
})
return {
"total_input_tokens": total_input,
"total_output_tokens": total_output,
"total_tokens": total_total,
"per_seat": per_seat,
}