董宏凯andCursor
1a080311bd
Make ensure_participants concurrent-safe, restore clarification checkpoints without re-planning, and only create threads for validated seats. Co-authored-by: Cursor <cursoragent@cursor.com>
943 lines
42 KiB
Python
943 lines
42 KiB
Python
"""MAF Workflow runner — orchestrates workflow execution for a MAF run.
|
|
|
|
Called after ``create_run`` to actually execute the workflow. Translates all
|
|
MAF Workflow events into ``maf_roundtable_events`` (the single event stream
|
|
the frontend reads). Manages run status lifecycle and checkpoint persistence.
|
|
|
|
There is NO sequential fallback. If the MAF SDK is missing or the workflow
|
|
fails, the run is marked ``failed`` with a sanitized error message. The runner
|
|
never pretends a workflow executed when it did not.
|
|
|
|
Usage::
|
|
|
|
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
|
await runner.execute(session, run)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from app.maf_roundtable.action_coordinator import ActionCoordinator
|
|
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
|
|
from app.maf_roundtable.fact_ledger import record_action_transition
|
|
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
|
from app.maf_roundtable.sanitize import sanitize
|
|
from app.maf_roundtable.workflow_executors import (
|
|
ApprovalGateExecutor,
|
|
CoordinatorExecutor,
|
|
CrossReviewExecutor,
|
|
DispatchPreparationTerminalExecutor,
|
|
DispatchSeatEntryExecutor,
|
|
FinalizerExecutor,
|
|
ManagerAnswerExecutor,
|
|
ManagerDispatchExecutor,
|
|
PlannerExecutor,
|
|
RoundtableState,
|
|
SeatExecutor,
|
|
WaitingTerminalExecutor,
|
|
_agent_capabilities_from_snapshot,
|
|
generate_final_report,
|
|
)
|
|
from app.maf_roundtable.workflow_factory import build_manager_dispatch_workflow, create_workflow
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MafSdkMissingError(RuntimeError):
|
|
"""Raised when the MAF SDK (agent_framework) is not installed."""
|
|
|
|
|
|
class WorkflowRunner:
|
|
"""Executes a MAF workflow for a given session/run pair."""
|
|
|
|
def __init__(self, app: Any, *, service: Any, user_id: str) -> None:
|
|
self._app = app
|
|
self._service = service
|
|
self._user_id = user_id
|
|
self._client = DeerFlowRuntimeClient(app)
|
|
self._adapter = DeerFlowAgentAdapter(self._client, service=service, user_id=user_id)
|
|
self._action_coordinator = ActionCoordinator(service, user_id=user_id)
|
|
|
|
@property
|
|
def adapter(self) -> DeerFlowAgentAdapter:
|
|
return self._adapter
|
|
|
|
@property
|
|
def action_coordinator(self) -> ActionCoordinator:
|
|
return self._action_coordinator
|
|
|
|
async def execute(self, session: dict[str, Any], run: dict[str, Any]) -> None:
|
|
"""Execute the full workflow for a session/run.
|
|
|
|
Transitions run status: pending → running → completed/failed/cancelled.
|
|
Persists checkpoint (RoundtableState) after each major step so the
|
|
workflow can resume after approval or restart.
|
|
"""
|
|
run_id = run["id"]
|
|
session_id = session["id"]
|
|
t0 = time.perf_counter()
|
|
|
|
# Verify the MAF SDK is available before touching anything.
|
|
try:
|
|
import agent_framework # noqa: F401
|
|
except ImportError as exc:
|
|
await self._fail_run(run_id, session_id, t0, "MafSdkMissing", "agent_framework SDK is not installed")
|
|
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
|
|
|
|
await self._service.store.runs.update_run(run_id, {
|
|
"status": "running",
|
|
"started_at": datetime.now(UTC),
|
|
})
|
|
|
|
try:
|
|
participants = await self._service.store.participants.list_participants(session_id)
|
|
|
|
task_payload = session.get("task_payload") or {}
|
|
maf_v2_payload = task_payload.get("maf_v2") if isinstance(task_payload, dict) else {}
|
|
maf_v2_payload = maf_v2_payload if isinstance(maf_v2_payload, dict) else {}
|
|
candidate_set = maf_v2_payload.get("candidate_set")
|
|
candidate_set = candidate_set if isinstance(candidate_set, dict) else {}
|
|
|
|
# P1-3: determine dispatch_contract_enabled before deciding which
|
|
# participants should receive threads on the first ensure_threads call.
|
|
dispatch_contract_enabled = bool(
|
|
maf_v2_payload.get("dispatch_contract_enabled")
|
|
or candidate_set
|
|
or maf_v2_payload.get("dispatch_plan") is not None
|
|
)
|
|
|
|
thread_map = await self._adapter.ensure_threads(
|
|
session_id,
|
|
run_id,
|
|
self._participants_requiring_threads(
|
|
participants,
|
|
dispatch_contract_enabled=dispatch_contract_enabled,
|
|
dispatch_plan=maf_v2_payload.get("dispatch_plan"),
|
|
intent_type=maf_v2_payload.get("intent_type"),
|
|
),
|
|
)
|
|
|
|
persisted_seat_agents = [
|
|
p["agent_id"] for p in participants if p.get("role") != "manager"
|
|
]
|
|
manager_id = session.get("manager_agent_id")
|
|
seat_timeout, schema_repair = self._orchestration_settings()
|
|
task_payload = session.get("task_payload") or {}
|
|
maf_v2_payload = task_payload.get("maf_v2") if isinstance(task_payload, dict) else {}
|
|
maf_v2_payload = maf_v2_payload if isinstance(maf_v2_payload, dict) else {}
|
|
candidate_set = maf_v2_payload.get("candidate_set")
|
|
candidate_set = candidate_set if isinstance(candidate_set, dict) else {}
|
|
candidate_seat_agents = list(
|
|
candidate_set.get("candidate_seat_agent_ids")
|
|
or maf_v2_payload.get("candidate_seat_agent_ids")
|
|
or (maf_v2_payload.get("dispatch_plan") or {}).get("candidate_seat_agent_ids")
|
|
or persisted_seat_agents
|
|
)
|
|
if not manager_id:
|
|
manager_id = candidate_set.get("manager_agent_id") or maf_v2_payload.get(
|
|
"manager_agent_id"
|
|
)
|
|
|
|
state = RoundtableState(
|
|
session_id=session_id,
|
|
run_id=run_id,
|
|
task_text=session.get("task_text", ""),
|
|
mode=session.get("mode", "structured"),
|
|
model_name=session.get("model_name"),
|
|
reasoning_effort=session.get("reasoning_effort"),
|
|
max_rounds=session.get("max_rounds", 8),
|
|
write_policy=session.get("write_policy", "confirm_high_risk"),
|
|
manager_agent_id=manager_id,
|
|
seat_agent_ids=candidate_seat_agents,
|
|
thread_map=thread_map,
|
|
capability_snapshot=session.get("capability_snapshot"),
|
|
seat_timeout_seconds=seat_timeout,
|
|
schema_repair_enabled=schema_repair,
|
|
intent_type=maf_v2_payload.get("intent_type"),
|
|
dispatch_plan=maf_v2_payload.get("dispatch_plan"),
|
|
candidate_seat_agent_ids=candidate_seat_agents,
|
|
dispatch_contract_enabled=bool(
|
|
maf_v2_payload.get("dispatch_contract_enabled")
|
|
or candidate_set
|
|
or maf_v2_payload.get("dispatch_plan") is not None
|
|
),
|
|
)
|
|
|
|
if state.dispatch_contract_enabled:
|
|
await self._run_manager_dispatch(state)
|
|
if state.status != "waiting_input":
|
|
workflow = self._build_workflow(state)
|
|
await self._run_workflow(workflow, state)
|
|
else:
|
|
workflow = self._build_workflow(state)
|
|
await self._run_workflow(workflow, state)
|
|
|
|
# Persist final state. waiting_approval is NOT a terminal state —
|
|
# it must NOT write finished_at. Only completed/failed/cancelled do.
|
|
latency = int((time.perf_counter() - t0) * 1000)
|
|
final_status = state.status
|
|
token_usage = _aggregate_token_usage(state.turn_results)
|
|
|
|
update_fields: dict[str, Any] = {
|
|
"status": final_status,
|
|
"latency_ms": latency,
|
|
"maf_checkpoint": state.to_dict(),
|
|
"token_usage": token_usage,
|
|
}
|
|
# Only write finished_at for terminal statuses.
|
|
if final_status in ("completed", "failed", "cancelled"):
|
|
update_fields["finished_at"] = datetime.now(UTC)
|
|
|
|
await self._service.store.runs.update_run(run_id, update_fields)
|
|
|
|
session_fields: dict[str, Any] = {"status": final_status}
|
|
if final_status == "completed":
|
|
session_fields["final_result"] = state.final_result
|
|
if final_status != "waiting_approval":
|
|
session_fields["fact_ledger"] = state.fact_ledger
|
|
await self._service.store.sessions.update_session(
|
|
session_id, user_id=self._user_id, fields=session_fields
|
|
)
|
|
|
|
except MafSdkMissingError:
|
|
raise
|
|
except Exception as exc:
|
|
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
|
|
|
|
def _orchestration_settings(self) -> tuple[float, bool]:
|
|
try:
|
|
from deerflow.config import get_app_config
|
|
|
|
cfg = get_app_config().maf_roundtable
|
|
return (
|
|
float(getattr(cfg, "seat_timeout_seconds", 600) or 600),
|
|
bool(getattr(cfg, "schema_repair_enabled", True)),
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
return 600.0, True
|
|
|
|
def _participants_requiring_threads(
|
|
self,
|
|
participants: list[dict[str, Any]],
|
|
*,
|
|
dispatch_contract_enabled: bool,
|
|
dispatch_plan: dict[str, Any] | None,
|
|
intent_type: str | None,
|
|
) -> list[dict[str, Any]]:
|
|
"""Determine which participants should receive threads at startup.
|
|
|
|
P1-3: Old V2 sessions may have persisted ALL candidate seats before
|
|
dispatch was validated. This method restricts the initial
|
|
``ensure_threads`` call so that unvalidated seats never receive a
|
|
thread, enforcing the rule "only the manager gets a thread until the
|
|
dispatch plan is server-validated".
|
|
|
|
Rules:
|
|
┌─────────────────────────────────────────────────────────────────────┐
|
|
│ State │ Allowed participants │
|
|
├──────────────────────────────┼──────────────────────────────────────┤
|
|
│ New V2, not yet dispatched │ manager only │
|
|
│ Validated direct_answer │ manager only │
|
|
│ Validated clarification │ manager only (usually no-op) │
|
|
│ Validated normal dispatch │ manager + selected seats from plan │
|
|
│ Legacy MAF (no V2 contract) │ all participants (original behavior) │
|
|
└──────────────────────────────┴──────────────────────────────────────┘
|
|
"""
|
|
# Legacy MAF sessions without V2 contract keep original behavior.
|
|
if not dispatch_contract_enabled:
|
|
return participants
|
|
|
|
manager_participants = [
|
|
p for p in participants if p.get("role") == "manager"
|
|
]
|
|
|
|
# No validated plan yet → only manager gets a thread.
|
|
if not dispatch_plan:
|
|
return manager_participants
|
|
|
|
plan_intent = str(dispatch_plan.get("intent_type") or intent_type or "")
|
|
execution_mode = str(dispatch_plan.get("execution_mode") or "")
|
|
|
|
# direct_answer or clarification → manager only.
|
|
if plan_intent in ("direct_answer", "clarification") or execution_mode == "manager_only":
|
|
return manager_participants
|
|
|
|
# Validated dispatch with selected seats → manager + those seats.
|
|
selected_ids = set(dispatch_plan.get("seat_agent_ids") or [])
|
|
if not selected_ids:
|
|
return manager_participants
|
|
|
|
return [
|
|
p for p in participants
|
|
if p.get("role") == "manager" or p.get("agent_id") in selected_ids
|
|
]
|
|
|
|
async def _run_manager_dispatch(self, state: RoundtableState) -> None:
|
|
"""Freeze the V2 manager decision before constructing seat topology."""
|
|
if state.is_completed("manager_dispatch"):
|
|
return
|
|
workflow = build_manager_dispatch_workflow(
|
|
manager_dispatch=ManagerDispatchExecutor(self._adapter, state),
|
|
terminal=DispatchPreparationTerminalExecutor(state),
|
|
)
|
|
await self._run_workflow(workflow, state)
|
|
|
|
def _build_workflow(self, state: RoundtableState) -> Any:
|
|
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
|
finalizer = FinalizerExecutor(self._adapter, state)
|
|
waiting_terminal = WaitingTerminalExecutor()
|
|
|
|
# New V2 conversations always start at the same constrained manager
|
|
# dispatch node. The manager preparation workflow finishes before this
|
|
# method is entered, so topology is selected from the persisted plan,
|
|
# never from a route default or a model-supplied agent id.
|
|
if state.dispatch_contract_enabled:
|
|
manager_answer = ManagerAnswerExecutor(self._adapter, state)
|
|
plan = state.dispatch_plan or {}
|
|
execution_mode = str(plan.get("execution_mode") or "")
|
|
selected_seat_ids = list(state.selected_seat_agent_ids or [])
|
|
|
|
if state.intent_type == "clarification" or not execution_mode:
|
|
raise RuntimeError("V2 manager dispatch did not produce a valid execution plan")
|
|
|
|
if execution_mode == "manager_only":
|
|
return create_workflow(
|
|
mode="manager_only",
|
|
manager_answer=manager_answer,
|
|
seat_executors=[],
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=[],
|
|
)
|
|
|
|
if state.mode == "group_chat":
|
|
coordinator = CoordinatorExecutor(self._adapter, state)
|
|
return create_workflow(
|
|
mode="group_chat",
|
|
coordinator=coordinator,
|
|
seat_executors=[
|
|
SeatExecutor(
|
|
self._adapter,
|
|
state,
|
|
agent_id,
|
|
action_coordinator=self._action_coordinator,
|
|
)
|
|
for agent_id in selected_seat_ids
|
|
],
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=selected_seat_ids,
|
|
)
|
|
|
|
state.current_round = max(state.current_round, 1)
|
|
cross_review = CrossReviewExecutor(self._adapter, state)
|
|
return create_workflow(
|
|
mode="structured",
|
|
planner=DispatchSeatEntryExecutor(state),
|
|
cross_review=cross_review,
|
|
seat_executors=[
|
|
SeatExecutor(
|
|
self._adapter,
|
|
state,
|
|
agent_id,
|
|
action_coordinator=self._action_coordinator,
|
|
)
|
|
for agent_id in selected_seat_ids
|
|
],
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=selected_seat_ids,
|
|
execution_mode=execution_mode,
|
|
)
|
|
|
|
seat_executors = [
|
|
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
|
for agent_id in state.seat_agent_ids
|
|
]
|
|
execution_mode = str(
|
|
(state.dispatch_plan or {}).get("execution_mode")
|
|
or ("manager_only" if state.intent_type == "direct_answer" else "parallel")
|
|
)
|
|
if execution_mode == "manager_only":
|
|
manager_answer = ManagerAnswerExecutor(self._adapter, state)
|
|
return create_workflow(
|
|
mode="manager_only",
|
|
manager_answer=manager_answer,
|
|
seat_executors=[],
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=[],
|
|
)
|
|
|
|
if state.mode == "group_chat":
|
|
coordinator = CoordinatorExecutor(self._adapter, state)
|
|
return create_workflow(
|
|
mode=state.mode,
|
|
coordinator=coordinator,
|
|
seat_executors=seat_executors,
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=state.seat_agent_ids,
|
|
)
|
|
|
|
planner = PlannerExecutor(self._adapter, state)
|
|
cross_review = CrossReviewExecutor(self._adapter, state)
|
|
return create_workflow(
|
|
mode="structured",
|
|
planner=planner,
|
|
cross_review=cross_review,
|
|
seat_executors=seat_executors,
|
|
approval_gate=approval_gate,
|
|
finalizer=finalizer,
|
|
waiting_terminal=waiting_terminal,
|
|
seat_agent_ids=state.seat_agent_ids,
|
|
execution_mode=execution_mode,
|
|
)
|
|
|
|
async def _persist_checkpoint(self, state: RoundtableState, *, reason: str = "") -> None:
|
|
await self._service.store.runs.update_run(state.run_id, {
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
|
|
|
await self._adapter._emit_critical(
|
|
state.run_id,
|
|
make_workflow_checkpointed(
|
|
None,
|
|
state.current_round,
|
|
checkpoint_id=f"nodes-{len(state.completed_nodes)}",
|
|
reason=reason or f"stage={state.stage}",
|
|
),
|
|
)
|
|
|
|
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
|
|
"""Execute the workflow graph via the MAF SDK's streaming API.
|
|
|
|
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
|
|
events. Persists the RoundtableState checkpoint after each superstep.
|
|
"""
|
|
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
|
|
event_type = getattr(event, "type", "")
|
|
executor_id = getattr(event, "executor_id", None)
|
|
|
|
if event_type == "superstep_completed":
|
|
iteration = getattr(event, "iteration", None)
|
|
await self._persist_checkpoint(
|
|
state, reason=f"superstep_completed:{iteration}"
|
|
)
|
|
|
|
elif event_type in ("executor_completed", "executor_invoked"):
|
|
# Persist after each node so completed_nodes survive restarts.
|
|
if state.completed_nodes:
|
|
await self._service.store.runs.update_run(state.run_id, {
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
|
|
elif event_type == "executor_failed":
|
|
details = getattr(event, "details", None)
|
|
error_msg = str(details) if details else "executor failed"
|
|
# Soft seat failures are handled inside SeatExecutor; only hard
|
|
# failures (e.g. WriteGuardConfigurationError) abort the run.
|
|
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
|
|
|
|
# Always persist latest completed_nodes at end of stream.
|
|
await self._service.store.runs.update_run(state.run_id, {
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
|
|
# Check if the workflow ended in a waiting_approval state.
|
|
if state.status == "waiting_approval" or state.pending_approvals:
|
|
state.status = "waiting_approval"
|
|
|
|
async def _fail_run(self, run_id: str, session_id: str, t0: float, error_code: str, error_message: str) -> None:
|
|
"""Mark a run as failed with a sanitized error message."""
|
|
latency = int((time.perf_counter() - t0) * 1000)
|
|
safe_message = str(sanitize(error_message))[:500]
|
|
logger.error("MAF workflow failed for run %s: %s", run_id, safe_message, exc_info=True)
|
|
await self._service.store.runs.update_run(run_id, {
|
|
"status": "failed",
|
|
"finished_at": datetime.now(UTC),
|
|
"error_code": error_code,
|
|
"error_message_safe": safe_message,
|
|
"latency_ms": latency,
|
|
})
|
|
await self._service.store.sessions.update_session(session_id, user_id=self._user_id, fields={
|
|
"status": "failed",
|
|
})
|
|
|
|
async def cancel(self, run_id: str) -> dict[str, Any]:
|
|
"""Cancel a running workflow (cascades to all active seat runs).
|
|
|
|
Returns the structured cancel result from the adapter so the caller
|
|
can decide whether to finalize the parent run as ``cancelled`` or
|
|
leave it non-terminal when some sub-runs failed to cancel.
|
|
"""
|
|
return await self._adapter.cancel(run_id)
|
|
|
|
async def resume(self, run_id: str) -> dict[str, Any] | None:
|
|
"""Resume a workflow from its persisted checkpoint after an approval decision.
|
|
|
|
P1: ``RoundtableState.from_dict()`` is actually called here. Flow:
|
|
|
|
1. Load ``maf_checkpoint_json`` from the run row.
|
|
2. Restore ``RoundtableState`` via ``from_dict()``.
|
|
3. Check the pending approvals' status:
|
|
- ``approved``: execute the approved action via ToolExecutor
|
|
(at-most-once, deterministic, no LLM).
|
|
- ``rejected`` / ``expired``: write ``action.rejected``, skip.
|
|
- ``pending``: do NOT start a second Workflow — return immediately.
|
|
4. After all actions are executed, determine the parent run status:
|
|
- All succeeded → ``completed``.
|
|
- Any failed → ``failed`` (P1-8).
|
|
- Any unknown → ``waiting_input`` (not completed; human reconcile needed).
|
|
- All skipped (already executed) → ``completed``.
|
|
|
|
Only the owner can resume (enforced by the router before calling
|
|
``dispatcher.resume``).
|
|
"""
|
|
t0 = time.perf_counter()
|
|
run = await self._service.store.runs.get_run(run_id)
|
|
if run is None:
|
|
return None
|
|
|
|
if run["status"] != "waiting_approval":
|
|
return run
|
|
|
|
checkpoint = run.get("maf_checkpoint")
|
|
if not checkpoint:
|
|
logger.warning("Cannot resume run %s: no checkpoint", run_id)
|
|
return run
|
|
|
|
state = RoundtableState.from_dict(checkpoint)
|
|
session_id = state.session_id
|
|
|
|
if not state.pending_approvals:
|
|
logger.warning("Cannot resume run %s: no pending_approvals in checkpoint", run_id)
|
|
return run
|
|
|
|
# Expire due approvals before inspecting the checkpoint. This is a
|
|
# fail-closed server transition; a late HTTP approval cannot revive it.
|
|
await self._service.store.expire_due_approvals(run_id=run_id)
|
|
|
|
# Check if any approval is still pending — if so, don't resume yet.
|
|
any_pending = False
|
|
for pa in state.pending_approvals:
|
|
approval_id = pa.get("approval_id", "")
|
|
approval = await self._service.store.approvals.get_approval(approval_id)
|
|
if approval and approval.get("status") == "pending":
|
|
any_pending = True
|
|
if any_pending:
|
|
return run
|
|
|
|
# P0-3 run-level CAS: only ONE concurrent resume may execute the
|
|
# approved actions and write the parent run's final state. The winner
|
|
# moves the run waiting_approval → running; every other concurrent
|
|
# caller loses the CAS and returns the current run WITHOUT writing a
|
|
# terminal status (so it can never race a completed/failed write).
|
|
claimed_run = await self._service.store.runs.claim_for_resume(run_id)
|
|
if not claimed_run:
|
|
logger.info("resume(%s): lost run-level CAS — another resume is executing", run_id)
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
# Execute each approved action (skip rejected/expired).
|
|
action_outcomes: list[dict[str, Any]] = []
|
|
for pa in state.pending_approvals:
|
|
approval_id = pa.get("approval_id", "")
|
|
action_id = pa.get("action_id", "")
|
|
# Extracted up-front so EVERY branch (rejected / failed / approved)
|
|
# can write the ledger transition with the correct agent_id.
|
|
agent_id = pa.get("agent_id", "")
|
|
thread_id = state.thread_map.get(agent_id, "")
|
|
tool_name = pa.get("tool_name", "")
|
|
approval = await self._service.store.approvals.get_approval(approval_id)
|
|
if approval is None:
|
|
action_outcomes.append({"action_id": action_id, "status": "failed",
|
|
"error": "approval not found"})
|
|
record_action_transition(
|
|
state.fact_ledger,
|
|
action_id=action_id,
|
|
tool_name=pa.get("tool_name", ""),
|
|
status="failed",
|
|
agent_id=agent_id,
|
|
)
|
|
continue
|
|
|
|
approval_status = approval.get("status", "pending")
|
|
|
|
if approval_status in ("rejected", "expired"):
|
|
from app.maf_roundtable.event_normalizer import make_action_rejected
|
|
# Conditional transition (waiting_approval → rejected) — never
|
|
# overwrite a terminal state via a generic update.
|
|
changed = await self._service.store.actions.mark_rejected(action_id)
|
|
if changed:
|
|
await self._adapter._emit_critical(run_id, make_action_rejected(
|
|
pa.get("agent_id", ""),
|
|
state.current_round,
|
|
action_id=action_id,
|
|
approval_id=approval_id,
|
|
decided_by=approval.get("decided_by", ""),
|
|
))
|
|
action_outcomes.append({"action_id": action_id, "status": "rejected"})
|
|
# Phase-3 P1-3: move the ledger row out of pending into the
|
|
# failed/unknown partition as rejected.
|
|
record_action_transition(
|
|
state.fact_ledger,
|
|
action_id=action_id,
|
|
tool_name=pa.get("tool_name", ""),
|
|
status="rejected",
|
|
agent_id=agent_id,
|
|
)
|
|
continue
|
|
|
|
if approval_status != "approved":
|
|
action_outcomes.append({"action_id": action_id, "status": "failed",
|
|
"error": f"unexpected approval status: {approval_status}"})
|
|
record_action_transition(
|
|
state.fact_ledger,
|
|
action_id=action_id,
|
|
tool_name=pa.get("tool_name", ""),
|
|
status="failed",
|
|
agent_id=agent_id,
|
|
)
|
|
continue
|
|
|
|
# Approved — execute deterministically via ToolExecutor.
|
|
# (agent_id / thread_id / tool_name are already extracted at the
|
|
# top of this loop iteration so every branch can use them.)
|
|
|
|
# Conditional transition (waiting_approval → approved). The
|
|
# authoritative at-most-once gate is the ToolExecutor's
|
|
# approved → running claim below; this just advances the state
|
|
# machine without ever clobbering a terminal status.
|
|
await self._service.store.actions.mark_approved(action_id)
|
|
|
|
from app.maf_roundtable.event_normalizer import make_action_approved
|
|
await self._adapter._emit_critical(run_id, make_action_approved(
|
|
agent_id,
|
|
state.current_round,
|
|
action_id=action_id,
|
|
approval_id=approval_id,
|
|
decided_by=approval.get("decided_by", ""),
|
|
))
|
|
|
|
# Params are recovered server-side inside ToolExecutor (after the
|
|
# atomic occupy). A missing/undecryptable secret fails the action
|
|
# there — it is never coerced into empty params, so the tool is
|
|
# never called (execution count == 0) and the parent run fails.
|
|
if thread_id and tool_name:
|
|
from app.maf_roundtable.tool_executor import ToolExecutor
|
|
|
|
executor = ToolExecutor(
|
|
adapter=self._adapter,
|
|
action_coordinator=self._action_coordinator,
|
|
user_id=self._user_id,
|
|
)
|
|
try:
|
|
outcome = await executor.execute(
|
|
action_id=action_id,
|
|
session_id=session_id,
|
|
run_id=run_id,
|
|
agent_id=agent_id,
|
|
thread_id=thread_id,
|
|
approved_tool=tool_name,
|
|
capability_snapshot=state.capability_snapshot,
|
|
write_policy=state.write_policy,
|
|
model_name=state.model_name,
|
|
excluded_tools=list(
|
|
_agent_capabilities_from_snapshot(
|
|
state.capability_snapshot, agent_id
|
|
).get("excluded_tools")
|
|
or []
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
safe_msg = str(sanitize(str(exc)))[:500]
|
|
logger.error("ToolExecutor for action %s failed: %s", action_id, safe_msg)
|
|
outcome = {"status": "failed", "error": safe_msg}
|
|
else:
|
|
outcome = {"status": "failed", "error": f"no thread_id or tool_name for action {action_id}"}
|
|
|
|
action_outcomes.append({
|
|
"action_id": action_id,
|
|
"tool_name": tool_name,
|
|
"status": outcome.get("status"),
|
|
"error": outcome.get("error"),
|
|
})
|
|
|
|
out_status = outcome.get("status")
|
|
ledger_status = {
|
|
"succeeded": "executed",
|
|
"failed": "failed",
|
|
"unknown": "unknown",
|
|
"skipped": "executed",
|
|
}.get(str(out_status), "unknown")
|
|
record_action_transition(
|
|
state.fact_ledger,
|
|
action_id=action_id,
|
|
tool_name=tool_name,
|
|
status=ledger_status,
|
|
agent_id=agent_id,
|
|
)
|
|
|
|
# P0-3: determine parent run status from action outcomes.
|
|
statuses = {o.get("status") for o in action_outcomes}
|
|
|
|
# A target held by another approved write remains pending for an
|
|
# explicit later resume. It is never retried automatically.
|
|
if "waiting_target_lease" in statuses:
|
|
from app.maf_roundtable.event_normalizer import make_action_waiting_target_lease
|
|
|
|
state.status = "waiting_approval"
|
|
for outcome in action_outcomes:
|
|
if outcome.get("status") == "waiting_target_lease":
|
|
await self._adapter._emit_critical(
|
|
run_id,
|
|
make_action_waiting_target_lease(
|
|
next(
|
|
(str(pa.get("agent_id") or "") for pa in state.pending_approvals if pa.get("action_id") == outcome.get("action_id")),
|
|
"",
|
|
),
|
|
state.current_round,
|
|
action_id=str(outcome.get("action_id") or ""),
|
|
tool_name=str(outcome.get("tool_name") or ""),
|
|
),
|
|
)
|
|
await self._service.store.runs.update_run(run_id, {
|
|
"status": "waiting_approval",
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
await self._service.store.sessions.update_session(
|
|
session_id,
|
|
user_id=self._user_id,
|
|
fields={"status": "waiting_approval", "fact_ledger": state.fact_ledger},
|
|
)
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
# While any action is still in_progress (another executor running it),
|
|
# the parent run stays running — never write finished_at, never finalize.
|
|
if "in_progress" in statuses:
|
|
state.status = "running"
|
|
await self._service.store.runs.update_run(run_id, {
|
|
"status": "running",
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
logger.info("resume(%s): action(s) still in_progress — run stays running", run_id)
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
state.pending_approvals = []
|
|
|
|
has_unknown = "unknown" in statuses
|
|
has_failed = "failed" in statuses
|
|
all_resolved = statuses <= {"succeeded", "skipped", "rejected"}
|
|
|
|
if has_unknown:
|
|
state.status = "waiting_input" # human reconciliation needed
|
|
elif has_failed:
|
|
state.status = "failed"
|
|
elif all_resolved:
|
|
# Re-enter the MAF finalizer path to synthesize the real report —
|
|
# do NOT just flip the status to completed.
|
|
try:
|
|
await generate_final_report(self._adapter, state)
|
|
state.status = "completed"
|
|
except Exception as exc: # noqa: BLE001
|
|
safe_msg = str(sanitize(str(exc)))[:500]
|
|
logger.error("resume(%s): finalizer failed: %s", run_id, safe_msg)
|
|
state.status = "failed"
|
|
action_outcomes.append({"action_id": "", "status": "failed",
|
|
"error": f"finalizer failed: {safe_msg}"})
|
|
else:
|
|
state.status = "failed"
|
|
|
|
# Persist final state.
|
|
latency = int((time.perf_counter() - t0) * 1000)
|
|
token_usage = _aggregate_token_usage(state.turn_results)
|
|
update_fields: dict[str, Any] = {
|
|
"status": state.status,
|
|
"latency_ms": latency,
|
|
"maf_checkpoint": state.to_dict(),
|
|
"token_usage": token_usage,
|
|
}
|
|
if state.status in ("completed", "failed", "cancelled"):
|
|
update_fields["finished_at"] = datetime.now(UTC)
|
|
if state.status == "failed":
|
|
errors = [o.get("error") for o in action_outcomes if o.get("error")]
|
|
update_fields["error_code"] = "ActionExecutionFailed"
|
|
update_fields["error_message_safe"] = str(sanitize("; ".join(errors)))[:500]
|
|
|
|
await self._service.store.runs.update_run(run_id, update_fields)
|
|
|
|
session_fields: dict[str, Any] = {"status": state.status}
|
|
if state.status == "completed":
|
|
session_fields["final_result"] = state.final_result
|
|
session_fields["fact_ledger"] = state.fact_ledger
|
|
await self._service.store.sessions.update_session(
|
|
session_id, user_id=self._user_id, fields=session_fields
|
|
)
|
|
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
async def recover(self, run_id: str) -> dict[str, Any] | None:
|
|
"""Graph-level recovery after process restart (MAF-03-07).
|
|
|
|
Reloads ``RoundtableState`` from ``maf_checkpoint`` and re-enters the
|
|
structured workflow. Executors whose ids are in ``completed_nodes``
|
|
short-circuit without calling DeerFlow again.
|
|
"""
|
|
t0 = time.perf_counter()
|
|
try:
|
|
import agent_framework # noqa: F401
|
|
except ImportError as exc:
|
|
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
|
|
|
|
run = await self._service.store.runs.get_run(run_id)
|
|
if run is None:
|
|
return None
|
|
if run["status"] not in ("running", "waiting_approval"):
|
|
logger.info("recover(%s): status=%s — nothing to recover", run_id, run["status"])
|
|
return run
|
|
|
|
checkpoint = run.get("maf_checkpoint")
|
|
if not checkpoint:
|
|
logger.warning("recover(%s): no checkpoint", run_id)
|
|
return run
|
|
|
|
state = RoundtableState.from_dict(checkpoint)
|
|
session = await self._service.store.sessions.get_session(state.session_id, user_id=None)
|
|
if session is None:
|
|
return run
|
|
|
|
expired_actions = await self._service.store.recover_expired_action_leases(run_id=run_id)
|
|
if expired_actions:
|
|
from app.maf_roundtable.event_normalizer import make_action_unknown
|
|
|
|
expired_ids = {action["id"] for action in expired_actions}
|
|
for action in expired_actions:
|
|
record_action_transition(
|
|
state.fact_ledger,
|
|
action_id=action["id"],
|
|
tool_name=action.get("tool_name", ""),
|
|
status="unknown",
|
|
agent_id=action.get("agent_id", ""),
|
|
)
|
|
await self._adapter._emit_critical(
|
|
run_id,
|
|
make_action_unknown(
|
|
action.get("agent_id", ""),
|
|
int(action.get("turn_index") or state.current_round),
|
|
action_id=action["id"],
|
|
reason_code="target_lease_expired_during_execution",
|
|
),
|
|
)
|
|
state.pending_approvals = [
|
|
item for item in state.pending_approvals
|
|
if item.get("action_id") not in expired_ids
|
|
]
|
|
state.status = "waiting_input"
|
|
await self._service.store.runs.update_run(run_id, {
|
|
"status": "waiting_input",
|
|
"maf_checkpoint": state.to_dict(),
|
|
})
|
|
await self._service.store.sessions.update_session(
|
|
state.session_id,
|
|
user_id=self._user_id,
|
|
fields={"status": "waiting_input", "fact_ledger": state.fact_ledger},
|
|
)
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
# Ensure threads still bound — but only for participants that are
|
|
# allowed to have threads at this stage (P1-3).
|
|
participants = await self._service.store.participants.list_participants(state.session_id)
|
|
thread_map = await self._adapter.ensure_threads(
|
|
state.session_id,
|
|
run_id,
|
|
self._participants_requiring_threads(
|
|
participants,
|
|
dispatch_contract_enabled=state.dispatch_contract_enabled,
|
|
dispatch_plan=state.dispatch_plan,
|
|
intent_type=state.intent_type,
|
|
),
|
|
)
|
|
state.thread_map.update(thread_map)
|
|
|
|
if run["status"] == "waiting_approval":
|
|
# Approval path stays on the existing resume() — recover only
|
|
# continues orchestration for running graphs.
|
|
return run
|
|
|
|
await self._service.store.runs.update_run(run_id, {"status": "running"})
|
|
try:
|
|
if state.dispatch_contract_enabled:
|
|
await self._run_manager_dispatch(state)
|
|
if state.status != "waiting_input":
|
|
workflow = self._build_workflow(state)
|
|
await self._run_workflow(workflow, state)
|
|
else:
|
|
workflow = self._build_workflow(state)
|
|
await self._run_workflow(workflow, state)
|
|
|
|
latency = int((time.perf_counter() - t0) * 1000)
|
|
final_status = state.status
|
|
update_fields: dict[str, Any] = {
|
|
"status": final_status,
|
|
"latency_ms": latency,
|
|
"maf_checkpoint": state.to_dict(),
|
|
"token_usage": _aggregate_token_usage(state.turn_results),
|
|
}
|
|
if final_status in ("completed", "failed", "cancelled"):
|
|
update_fields["finished_at"] = datetime.now(UTC)
|
|
await self._service.store.runs.update_run(run_id, update_fields)
|
|
|
|
session_fields: dict[str, Any] = {"status": final_status}
|
|
if final_status == "completed":
|
|
session_fields["final_result"] = state.final_result
|
|
if final_status != "waiting_approval":
|
|
session_fields["fact_ledger"] = state.fact_ledger
|
|
await self._service.store.sessions.update_session(
|
|
state.session_id, user_id=self._user_id, fields=session_fields
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
await self._fail_run(run_id, state.session_id, t0, type(exc).__name__, str(exc))
|
|
|
|
return await self._service.store.runs.get_run(run_id)
|
|
|
|
|
|
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
|
|
"""Aggregate token usage across all seat turns into a run-level summary."""
|
|
total_input = 0
|
|
total_output = 0
|
|
total_total = 0
|
|
per_seat: list[dict[str, Any]] = []
|
|
for turn in turn_results:
|
|
usage = turn.get("usage")
|
|
if not isinstance(usage, dict):
|
|
continue
|
|
inp = usage.get("input_tokens", 0) or usage.get("prompt_tokens", 0) or 0
|
|
out = usage.get("output_tokens", 0) or usage.get("completion_tokens", 0) or 0
|
|
tot = usage.get("total_tokens", 0) or (inp + out)
|
|
total_input += inp
|
|
total_output += out
|
|
total_total += tot
|
|
per_seat.append({
|
|
"agent_id": turn.get("agent_id", ""),
|
|
"turn_index": turn.get("turn_index", 0),
|
|
"input_tokens": inp,
|
|
"output_tokens": out,
|
|
"total_tokens": tot,
|
|
})
|
|
return {
|
|
"total_input_tokens": total_input,
|
|
"total_output_tokens": total_output,
|
|
"total_tokens": total_total,
|
|
"per_seat": per_seat,
|
|
}
|