feat(maf): add secure roundtable workflow runtime
This commit is contained in:
1 parent
e690475e6e
commit
bd4131b4e2
52 files changed
+10315
-96
No files matched your search
@@ -0,0 +1,483 @@
|
||||
"""MAF Workflow runner — orchestrates workflow execution for a MAF run.
|
||||
|
||||
Called after ``create_run`` to actually execute the workflow. Translates all
|
||||
MAF Workflow events into ``maf_roundtable_events`` (the single event stream
|
||||
the frontend reads). Manages run status lifecycle and checkpoint persistence.
|
||||
|
||||
There is NO sequential fallback. If the MAF SDK is missing or the workflow
|
||||
fails, the run is marked ``failed`` with a sanitized error message. The runner
|
||||
never pretends a workflow executed when it did not.
|
||||
|
||||
Usage::
|
||||
|
||||
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
||||
await runner.execute(session, run)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.maf_roundtable.action_coordinator import ActionCoordinator
|
||||
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
|
||||
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
||||
from app.maf_roundtable.sanitize import sanitize
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CoordinatorExecutor,
|
||||
FinalizerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
WaitingTerminalExecutor,
|
||||
_agent_capabilities_from_snapshot,
|
||||
generate_final_report,
|
||||
)
|
||||
from app.maf_roundtable.workflow_factory import create_workflow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MafSdkMissingError(RuntimeError):
|
||||
"""Raised when the MAF SDK (agent_framework) is not installed."""
|
||||
|
||||
|
||||
class WorkflowRunner:
|
||||
"""Executes a MAF workflow for a given session/run pair."""
|
||||
|
||||
def __init__(self, app: Any, *, service: Any, user_id: str) -> None:
|
||||
self._app = app
|
||||
self._service = service
|
||||
self._user_id = user_id
|
||||
self._client = DeerFlowRuntimeClient(app)
|
||||
self._adapter = DeerFlowAgentAdapter(self._client, service=service, user_id=user_id)
|
||||
self._action_coordinator = ActionCoordinator(service, user_id=user_id)
|
||||
|
||||
@property
|
||||
def adapter(self) -> DeerFlowAgentAdapter:
|
||||
return self._adapter
|
||||
|
||||
@property
|
||||
def action_coordinator(self) -> ActionCoordinator:
|
||||
return self._action_coordinator
|
||||
|
||||
async def execute(self, session: dict[str, Any], run: dict[str, Any]) -> None:
|
||||
"""Execute the full workflow for a session/run.
|
||||
|
||||
Transitions run status: pending → running → completed/failed/cancelled.
|
||||
Persists checkpoint (RoundtableState) after each major step so the
|
||||
workflow can resume after approval or restart.
|
||||
"""
|
||||
run_id = run["id"]
|
||||
session_id = session["id"]
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# Verify the MAF SDK is available before touching anything.
|
||||
try:
|
||||
import agent_framework # noqa: F401
|
||||
except ImportError as exc:
|
||||
await self._fail_run(run_id, session_id, t0, "MafSdkMissing", "agent_framework SDK is not installed")
|
||||
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
|
||||
|
||||
await self._service.store.runs.update_run(run_id, {
|
||||
"status": "running",
|
||||
"started_at": datetime.now(UTC),
|
||||
})
|
||||
|
||||
try:
|
||||
participants = await self._service.store.participants.list_participants(session_id)
|
||||
thread_map = await self._adapter.ensure_threads(session_id, run_id, participants)
|
||||
|
||||
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
|
||||
manager_id = session.get("manager_agent_id")
|
||||
|
||||
state = RoundtableState(
|
||||
session_id=session_id,
|
||||
run_id=run_id,
|
||||
task_text=session.get("task_text", ""),
|
||||
mode=session.get("mode", "structured"),
|
||||
model_name=session.get("model_name"),
|
||||
reasoning_effort=session.get("reasoning_effort"),
|
||||
max_rounds=session.get("max_rounds", 8),
|
||||
write_policy=session.get("write_policy", "confirm_high_risk"),
|
||||
manager_agent_id=manager_id,
|
||||
seat_agent_ids=seat_agents,
|
||||
thread_map=thread_map,
|
||||
capability_snapshot=session.get("capability_snapshot"),
|
||||
)
|
||||
|
||||
# Build executor instances — one per seat, each bound to a fixed agent_id.
|
||||
coordinator = CoordinatorExecutor(self._adapter, state)
|
||||
seat_executors = [
|
||||
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
||||
for agent_id in seat_agents
|
||||
]
|
||||
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
||||
finalizer = FinalizerExecutor(self._adapter, state)
|
||||
waiting_terminal = WaitingTerminalExecutor()
|
||||
|
||||
workflow = create_workflow(
|
||||
mode=state.mode,
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=seat_agents,
|
||||
)
|
||||
|
||||
await self._run_workflow(workflow, state)
|
||||
|
||||
# Persist final state. waiting_approval is NOT a terminal state —
|
||||
# it must NOT write finished_at. Only completed/failed/cancelled do.
|
||||
latency = int((time.perf_counter() - t0) * 1000)
|
||||
final_status = state.status
|
||||
token_usage = _aggregate_token_usage(state.turn_results)
|
||||
|
||||
update_fields: dict[str, Any] = {
|
||||
"status": final_status,
|
||||
"latency_ms": latency,
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
"token_usage": token_usage,
|
||||
}
|
||||
# Only write finished_at for terminal statuses.
|
||||
if final_status in ("completed", "failed", "cancelled"):
|
||||
update_fields["finished_at"] = datetime.now(UTC)
|
||||
|
||||
await self._service.store.runs.update_run(run_id, update_fields)
|
||||
|
||||
session_fields: dict[str, Any] = {"status": final_status}
|
||||
if final_status == "completed":
|
||||
session_fields["final_result"] = state.final_result
|
||||
if final_status != "waiting_approval":
|
||||
session_fields["fact_ledger"] = state.fact_ledger
|
||||
await self._service.store.sessions.update_session(
|
||||
session_id, user_id=self._user_id, fields=session_fields
|
||||
)
|
||||
|
||||
except MafSdkMissingError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
|
||||
|
||||
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
|
||||
"""Execute the workflow graph via the MAF SDK's streaming API.
|
||||
|
||||
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
|
||||
events. Persists the RoundtableState checkpoint after each superstep.
|
||||
"""
|
||||
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
||||
|
||||
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
|
||||
event_type = getattr(event, "type", "")
|
||||
executor_id = getattr(event, "executor_id", None)
|
||||
|
||||
if event_type == "superstep_completed":
|
||||
# Persist checkpoint after each superstep.
|
||||
iteration = getattr(event, "iteration", None)
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
|
||||
executor_id, state.current_round,
|
||||
checkpoint_id=f"superstep-{iteration}",
|
||||
reason="superstep_completed",
|
||||
))
|
||||
|
||||
elif event_type == "executor_failed":
|
||||
details = getattr(event, "details", None)
|
||||
error_msg = str(details) if details else "executor failed"
|
||||
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
|
||||
|
||||
# Check if the workflow ended in a waiting_approval state.
|
||||
if state.status == "waiting_approval" or state.pending_approvals:
|
||||
state.status = "waiting_approval"
|
||||
|
||||
async def _fail_run(self, run_id: str, session_id: str, t0: float, error_code: str, error_message: str) -> None:
|
||||
"""Mark a run as failed with a sanitized error message."""
|
||||
latency = int((time.perf_counter() - t0) * 1000)
|
||||
safe_message = str(sanitize(error_message))[:500]
|
||||
logger.error("MAF workflow failed for run %s: %s", run_id, safe_message, exc_info=True)
|
||||
await self._service.store.runs.update_run(run_id, {
|
||||
"status": "failed",
|
||||
"finished_at": datetime.now(UTC),
|
||||
"error_code": error_code,
|
||||
"error_message_safe": safe_message,
|
||||
"latency_ms": latency,
|
||||
})
|
||||
await self._service.store.sessions.update_session(session_id, user_id=self._user_id, fields={
|
||||
"status": "failed",
|
||||
})
|
||||
|
||||
async def cancel(self, run_id: str) -> dict[str, Any]:
|
||||
"""Cancel a running workflow (cascades to all active seat runs).
|
||||
|
||||
Returns the structured cancel result from the adapter so the caller
|
||||
can decide whether to finalize the parent run as ``cancelled`` or
|
||||
leave it non-terminal when some sub-runs failed to cancel.
|
||||
"""
|
||||
return await self._adapter.cancel(run_id)
|
||||
|
||||
async def resume(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Resume a workflow from its persisted checkpoint after an approval decision.
|
||||
|
||||
P1: ``RoundtableState.from_dict()`` is actually called here. Flow:
|
||||
|
||||
1. Load ``maf_checkpoint_json`` from the run row.
|
||||
2. Restore ``RoundtableState`` via ``from_dict()``.
|
||||
3. Check the pending approvals' status:
|
||||
- ``approved``: execute the approved action via ToolExecutor
|
||||
(at-most-once, deterministic, no LLM).
|
||||
- ``rejected`` / ``expired``: write ``action.rejected``, skip.
|
||||
- ``pending``: do NOT start a second Workflow — return immediately.
|
||||
4. After all actions are executed, determine the parent run status:
|
||||
- All succeeded → ``completed``.
|
||||
- Any failed → ``failed`` (P1-8).
|
||||
- Any unknown → ``waiting_input`` (not completed; human reconcile needed).
|
||||
- All skipped (already executed) → ``completed``.
|
||||
|
||||
Only the owner can resume (enforced by the router before calling
|
||||
``dispatcher.resume``).
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
run = await self._service.store.runs.get_run(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
|
||||
if run["status"] != "waiting_approval":
|
||||
return run
|
||||
|
||||
checkpoint = run.get("maf_checkpoint")
|
||||
if not checkpoint:
|
||||
logger.warning("Cannot resume run %s: no checkpoint", run_id)
|
||||
return run
|
||||
|
||||
state = RoundtableState.from_dict(checkpoint)
|
||||
session_id = state.session_id
|
||||
|
||||
if not state.pending_approvals:
|
||||
logger.warning("Cannot resume run %s: no pending_approvals in checkpoint", run_id)
|
||||
return run
|
||||
|
||||
# Check if any approval is still pending — if so, don't resume yet.
|
||||
any_pending = False
|
||||
for pa in state.pending_approvals:
|
||||
approval_id = pa.get("approval_id", "")
|
||||
approval = await self._service.store.approvals.get_approval(approval_id)
|
||||
if approval and approval.get("status") == "pending":
|
||||
any_pending = True
|
||||
if any_pending:
|
||||
return run
|
||||
|
||||
# P0-3 run-level CAS: only ONE concurrent resume may execute the
|
||||
# approved actions and write the parent run's final state. The winner
|
||||
# moves the run waiting_approval → running; every other concurrent
|
||||
# caller loses the CAS and returns the current run WITHOUT writing a
|
||||
# terminal status (so it can never race a completed/failed write).
|
||||
claimed_run = await self._service.store.runs.claim_for_resume(run_id)
|
||||
if not claimed_run:
|
||||
logger.info("resume(%s): lost run-level CAS — another resume is executing", run_id)
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
# Execute each approved action (skip rejected/expired).
|
||||
action_outcomes: list[dict[str, Any]] = []
|
||||
for pa in state.pending_approvals:
|
||||
approval_id = pa.get("approval_id", "")
|
||||
action_id = pa.get("action_id", "")
|
||||
approval = await self._service.store.approvals.get_approval(approval_id)
|
||||
if approval is None:
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": "approval not found"})
|
||||
continue
|
||||
|
||||
approval_status = approval.get("status", "pending")
|
||||
|
||||
if approval_status in ("rejected", "expired"):
|
||||
from app.maf_roundtable.event_normalizer import make_action_rejected
|
||||
# Conditional transition (waiting_approval → rejected) — never
|
||||
# overwrite a terminal state via a generic update.
|
||||
await self._service.store.actions.mark_rejected(action_id)
|
||||
await self._adapter._emit_critical(run_id, make_action_rejected(
|
||||
pa.get("agent_id", ""),
|
||||
state.current_round,
|
||||
action_id=action_id,
|
||||
approval_id=approval_id,
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
action_outcomes.append({"action_id": action_id, "status": "rejected"})
|
||||
continue
|
||||
|
||||
if approval_status != "approved":
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": f"unexpected approval status: {approval_status}"})
|
||||
continue
|
||||
|
||||
# Approved — execute deterministically via ToolExecutor.
|
||||
agent_id = pa.get("agent_id", "")
|
||||
thread_id = state.thread_map.get(agent_id, "")
|
||||
tool_name = pa.get("tool_name", "")
|
||||
|
||||
# Conditional transition (waiting_approval → approved). The
|
||||
# authoritative at-most-once gate is the ToolExecutor's
|
||||
# approved → running claim below; this just advances the state
|
||||
# machine without ever clobbering a terminal status.
|
||||
await self._service.store.actions.mark_approved(action_id)
|
||||
|
||||
from app.maf_roundtable.event_normalizer import make_action_approved
|
||||
await self._adapter._emit_critical(run_id, make_action_approved(
|
||||
agent_id,
|
||||
state.current_round,
|
||||
action_id=action_id,
|
||||
approval_id=approval_id,
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
|
||||
# Params are recovered server-side inside ToolExecutor (after the
|
||||
# atomic occupy). A missing/undecryptable secret fails the action
|
||||
# there — it is never coerced into empty params, so the tool is
|
||||
# never called (execution count == 0) and the parent run fails.
|
||||
if thread_id and tool_name:
|
||||
from app.maf_roundtable.tool_executor import ToolExecutor
|
||||
|
||||
executor = ToolExecutor(
|
||||
adapter=self._adapter,
|
||||
action_coordinator=self._action_coordinator,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
try:
|
||||
outcome = await executor.execute(
|
||||
action_id=action_id,
|
||||
session_id=session_id,
|
||||
run_id=run_id,
|
||||
agent_id=agent_id,
|
||||
thread_id=thread_id,
|
||||
approved_tool=tool_name,
|
||||
capability_snapshot=state.capability_snapshot,
|
||||
write_policy=state.write_policy,
|
||||
model_name=state.model_name,
|
||||
excluded_tools=list(
|
||||
_agent_capabilities_from_snapshot(
|
||||
state.capability_snapshot, agent_id
|
||||
).get("excluded_tools")
|
||||
or []
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
safe_msg = str(sanitize(str(exc)))[:500]
|
||||
logger.error("ToolExecutor for action %s failed: %s", action_id, safe_msg)
|
||||
outcome = {"status": "failed", "error": safe_msg}
|
||||
else:
|
||||
outcome = {"status": "failed", "error": f"no thread_id or tool_name for action {action_id}"}
|
||||
|
||||
action_outcomes.append({
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"status": outcome.get("status"),
|
||||
"error": outcome.get("error"),
|
||||
})
|
||||
|
||||
state.fact_ledger["pending_actions"].append({
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"outcome": outcome.get("status"),
|
||||
})
|
||||
|
||||
# P0-3: determine parent run status from action outcomes.
|
||||
statuses = {o.get("status") for o in action_outcomes}
|
||||
|
||||
# While any action is still in_progress (another executor running it),
|
||||
# the parent run stays running — never write finished_at, never finalize.
|
||||
if "in_progress" in statuses:
|
||||
state.status = "running"
|
||||
await self._service.store.runs.update_run(run_id, {
|
||||
"status": "running",
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
logger.info("resume(%s): action(s) still in_progress — run stays running", run_id)
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
state.pending_approvals = []
|
||||
|
||||
has_unknown = "unknown" in statuses
|
||||
has_failed = "failed" in statuses
|
||||
all_resolved = statuses <= {"succeeded", "skipped", "rejected"}
|
||||
|
||||
if has_unknown:
|
||||
state.status = "waiting_input" # human reconciliation needed
|
||||
elif has_failed:
|
||||
state.status = "failed"
|
||||
elif all_resolved:
|
||||
# Re-enter the MAF finalizer path to synthesize the real report —
|
||||
# do NOT just flip the status to completed.
|
||||
try:
|
||||
await generate_final_report(self._adapter, state)
|
||||
state.status = "completed"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
safe_msg = str(sanitize(str(exc)))[:500]
|
||||
logger.error("resume(%s): finalizer failed: %s", run_id, safe_msg)
|
||||
state.status = "failed"
|
||||
action_outcomes.append({"action_id": "", "status": "failed",
|
||||
"error": f"finalizer failed: {safe_msg}"})
|
||||
else:
|
||||
state.status = "failed"
|
||||
|
||||
# Persist final state.
|
||||
latency = int((time.perf_counter() - t0) * 1000)
|
||||
token_usage = _aggregate_token_usage(state.turn_results)
|
||||
update_fields: dict[str, Any] = {
|
||||
"status": state.status,
|
||||
"latency_ms": latency,
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
"token_usage": token_usage,
|
||||
}
|
||||
if state.status in ("completed", "failed", "cancelled"):
|
||||
update_fields["finished_at"] = datetime.now(UTC)
|
||||
if state.status == "failed":
|
||||
errors = [o.get("error") for o in action_outcomes if o.get("error")]
|
||||
update_fields["error_code"] = "ActionExecutionFailed"
|
||||
update_fields["error_message_safe"] = str(sanitize("; ".join(errors)))[:500]
|
||||
|
||||
await self._service.store.runs.update_run(run_id, update_fields)
|
||||
|
||||
session_fields: dict[str, Any] = {"status": state.status}
|
||||
if state.status == "completed":
|
||||
session_fields["final_result"] = state.final_result
|
||||
session_fields["fact_ledger"] = state.fact_ledger
|
||||
await self._service.store.sessions.update_session(
|
||||
session_id, user_id=self._user_id, fields=session_fields
|
||||
)
|
||||
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
|
||||
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Aggregate token usage across all seat turns into a run-level summary."""
|
||||
total_input = 0
|
||||
total_output = 0
|
||||
total_total = 0
|
||||
per_seat: list[dict[str, Any]] = []
|
||||
for turn in turn_results:
|
||||
usage = turn.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
continue
|
||||
inp = usage.get("input_tokens", 0) or usage.get("prompt_tokens", 0) or 0
|
||||
out = usage.get("output_tokens", 0) or usage.get("completion_tokens", 0) or 0
|
||||
tot = usage.get("total_tokens", 0) or (inp + out)
|
||||
total_input += inp
|
||||
total_output += out
|
||||
total_total += tot
|
||||
per_seat.append({
|
||||
"agent_id": turn.get("agent_id", ""),
|
||||
"turn_index": turn.get("turn_index", 0),
|
||||
"input_tokens": inp,
|
||||
"output_tokens": out,
|
||||
"total_tokens": tot,
|
||||
})
|
||||
return {
|
||||
"total_input_tokens": total_input,
|
||||
"total_output_tokens": total_output,
|
||||
"total_tokens": total_total,
|
||||
"per_seat": per_seat,
|
||||
}
|
||||
Reference in new issue
Block a user