feat(maf): complete resilient roundtable workflow
This commit is contained in:
1 parent
bd4131b4e2
commit
21c4c898e7
23 files changed
+4810
-312
No files matched your search
@@ -127,11 +127,35 @@ def _format_prior_turns(turns: list[dict[str, Any]], max_per_turn: int) -> str:
|
||||
def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
|
||||
candidates = ledger.get("candidate_facts")
|
||||
if candidates and isinstance(candidates, list):
|
||||
# Phase-3: candidate facts are UNVERIFIED seat claims — label them
|
||||
# explicitly so downstream seats / the finalizer never treat them as
|
||||
# established conclusions.
|
||||
parts.append("未经核验的席位主张(不能作为结论):")
|
||||
for c in candidates[:20]:
|
||||
if isinstance(c, dict):
|
||||
parts.append(f" - {c.get('text', c)}")
|
||||
else:
|
||||
parts.append(f" - {c}")
|
||||
|
||||
facts = ledger.get("confirmed_facts")
|
||||
if facts and isinstance(facts, list):
|
||||
parts.append("已确认事实:")
|
||||
parts.append("已确认事实(经交叉审查核验):")
|
||||
for f in facts[:20]:
|
||||
parts.append(f" - {f}")
|
||||
if isinstance(f, dict):
|
||||
parts.append(f" - {f.get('text', f)}")
|
||||
else:
|
||||
parts.append(f" - {f}")
|
||||
|
||||
evidence = ledger.get("evidence")
|
||||
if evidence and isinstance(evidence, list):
|
||||
parts.append("证据引用:")
|
||||
for e in evidence[:10]:
|
||||
if isinstance(e, dict):
|
||||
parts.append(f" - {e.get('ref') or e.get('text') or e}")
|
||||
else:
|
||||
parts.append(f" - {e}")
|
||||
|
||||
tool_results = ledger.get("tool_results")
|
||||
if tool_results and isinstance(tool_results, list):
|
||||
@@ -142,6 +166,21 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
else:
|
||||
parts.append(f" - {tr}")
|
||||
|
||||
proposed = ledger.get("proposed_actions")
|
||||
if proposed and isinstance(proposed, list):
|
||||
parts.append("建议动作(未执行):")
|
||||
for p in proposed[:8]:
|
||||
parts.append(f" - {p}")
|
||||
|
||||
executed = ledger.get("executed_actions")
|
||||
if executed and isinstance(executed, list):
|
||||
parts.append("已执行外部写入:")
|
||||
for e in executed[:8]:
|
||||
if isinstance(e, dict):
|
||||
parts.append(f" - {e.get('tool_name', '?')} ({e.get('action_id', '')})")
|
||||
else:
|
||||
parts.append(f" - {e}")
|
||||
|
||||
conflicts = ledger.get("conflicts")
|
||||
if conflicts and isinstance(conflicts, list):
|
||||
parts.append("未解决冲突:")
|
||||
@@ -154,6 +193,12 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
for p in pending[:5]:
|
||||
parts.append(f" - {p}")
|
||||
|
||||
failed = ledger.get("failed_or_unknown_actions")
|
||||
if failed and isinstance(failed, list):
|
||||
parts.append("失败或状态不确定的动作:")
|
||||
for f in failed[:5]:
|
||||
parts.append(f" - {f}")
|
||||
|
||||
result = "\n".join(parts)
|
||||
if len(result) > max_chars:
|
||||
result = result[:max_chars]
|
||||
|
||||
@@ -13,7 +13,8 @@ Architecture:
|
||||
- Cancellation propagates from MAF run → DeerFlow run via RunManager.cancel.
|
||||
- All seat runs persisted to ``maf_roundtable_seat_runs`` (survives restart).
|
||||
- Critical audit events (tool.*, skill.*, approval.*, artifact.*) MUST persist;
|
||||
failure stops external writes and marks the run as failed.
|
||||
audit failure stops external writes; seat turn failures are soft-failed
|
||||
into ``seat_outcomes`` / ``partial_failure`` (structured mode).
|
||||
|
||||
Boundary: lives in ``app.maf_roundtable`` (may import ``deerflow.*`` and
|
||||
``app.gateway.*``); never imported by ``deerflow.*``.
|
||||
@@ -132,6 +133,7 @@ class DeerFlowAgentAdapter:
|
||||
turn_index: int,
|
||||
message: str,
|
||||
thread_id: str,
|
||||
execution_stage: str = "expert",
|
||||
model_name: str | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
thinking_enabled: bool = True,
|
||||
@@ -147,6 +149,7 @@ class DeerFlowAgentAdapter:
|
||||
capability_snapshot: dict[str, Any] | None = None,
|
||||
write_policy: str = "confirm_high_risk",
|
||||
approved_tool: str | None = None,
|
||||
wait_timeout_seconds: float | None = None,
|
||||
) -> SeatResponse:
|
||||
"""Execute one seat turn through the DeerFlow Agent Runtime."""
|
||||
t0 = time.perf_counter()
|
||||
@@ -166,10 +169,56 @@ class DeerFlowAgentAdapter:
|
||||
|
||||
principal = ExecutionPrincipal(user_id=self._user_id, session_id=session_id, run_id=run_id)
|
||||
|
||||
seat_run = await self._service.store.seat_runs.create_seat_run(
|
||||
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index, deerflow_thread_id=thread_id
|
||||
# Phase-3 P0-1: atomic "occupy or read". Only the caller that wins the
|
||||
# insert (created=True) may start a DeerFlow run. An existing row is
|
||||
# handled by status so we NEVER silently start a second DeerFlow
|
||||
# execution for the same four-tuple (run, agent, turn, stage).
|
||||
seat_run, created = await self._service.store.seat_runs.get_or_create_seat_run(
|
||||
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index,
|
||||
execution_stage=execution_stage, deerflow_thread_id=thread_id,
|
||||
)
|
||||
seat_run_id = seat_run["id"]
|
||||
prior_status = seat_run.get("status")
|
||||
|
||||
if not created:
|
||||
if prior_status == "completed":
|
||||
return self._seat_response_from_snapshot(
|
||||
seat_run=seat_run,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
if prior_status in ("pending", "running", "cancelling"):
|
||||
effective_timeout = wait_timeout_seconds if wait_timeout_seconds is not None else 600.0
|
||||
seat_run = await self._service.store.seat_runs.wait_for_terminal_seat_run(
|
||||
maf_run_id=run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
timeout_seconds=effective_timeout,
|
||||
)
|
||||
if seat_run["status"] == "completed":
|
||||
return self._seat_response_from_snapshot(
|
||||
seat_run=seat_run,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error=f"stage '{execution_stage}' ended as {seat_run['status']}",
|
||||
metadata={"seat_run_id": seat_run["id"], "terminal": seat_run["status"]},
|
||||
latency_ms=int((time.perf_counter() - t0) * 1000),
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error=f"stage '{execution_stage}' previously terminal ({prior_status}); "
|
||||
f"manual recovery required",
|
||||
metadata={"terminal": prior_status, "seat_run_id": seat_run_id},
|
||||
latency_ms=int((time.perf_counter() - t0) * 1000),
|
||||
)
|
||||
|
||||
try:
|
||||
response = await self._execute_run(
|
||||
@@ -210,7 +259,16 @@ class DeerFlowAgentAdapter:
|
||||
latency_ms=latency,
|
||||
)
|
||||
|
||||
await self._service.store.seat_runs.mark_completed(seat_run_id)
|
||||
raw_snapshot = {
|
||||
"content": response.content,
|
||||
"tool_calls": response.tool_calls,
|
||||
"usage": response.usage,
|
||||
"metadata": response.metadata,
|
||||
}
|
||||
snapshot = sanitize(raw_snapshot)
|
||||
if not isinstance(snapshot, dict):
|
||||
raise RuntimeError("seat result snapshot sanitization produced invalid payload")
|
||||
await self._service.store.seat_runs.mark_completed(seat_run_id, result_snapshot=snapshot)
|
||||
response.latency_ms = int((time.perf_counter() - t0) * 1000)
|
||||
await self._emit_critical(run_id, make_turn_completed(
|
||||
agent_id, turn_index,
|
||||
@@ -463,6 +521,34 @@ class DeerFlowAgentAdapter:
|
||||
|
||||
# ── internal helpers ────────────────────────────────────────────────────
|
||||
|
||||
def _seat_response_from_snapshot(
|
||||
self,
|
||||
*,
|
||||
seat_run: dict[str, Any],
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
) -> SeatResponse:
|
||||
snapshot = seat_run.get("result_snapshot")
|
||||
if not snapshot or not isinstance(snapshot, dict):
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error="completed seat run has no replayable snapshot; manual recovery required",
|
||||
metadata={"seat_run_id": seat_run.get("id"), "terminal": seat_run.get("status")},
|
||||
latency_ms=0,
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content=str(snapshot.get("content", "")),
|
||||
tool_calls=list(snapshot.get("tool_calls") or []),
|
||||
usage=snapshot.get("usage"),
|
||||
metadata=dict(snapshot.get("metadata") or {}),
|
||||
error=None,
|
||||
latency_ms=0,
|
||||
)
|
||||
|
||||
def _latest_ai_text(self, messages: list[dict[str, Any]]) -> str:
|
||||
for m in reversed(messages):
|
||||
if not isinstance(m, dict):
|
||||
|
||||
@@ -96,6 +96,36 @@ class MafRunDispatcher:
|
||||
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
|
||||
return await runner.resume(run_id)
|
||||
|
||||
async def recover(self, run_id: str, *, user_id: str) -> dict[str, Any] | None:
|
||||
"""Re-enter the workflow from checkpoint after a process restart.
|
||||
|
||||
Unlike ``resume`` (approval path), this continues orchestration for
|
||||
``running`` runs and skips ``completed_nodes``.
|
||||
"""
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
service = getattr(self._app.state, "maf_roundtable_service", None)
|
||||
if service is None:
|
||||
raise MafSdkNotAvailableError("MAF roundtable service not available")
|
||||
|
||||
# Avoid overlapping with an in-process execute task.
|
||||
existing = self._tasks.get(run_id)
|
||||
if existing is not None and not existing.done():
|
||||
logger.info("recover(%s): background task still running — skip", run_id)
|
||||
return await service.store.runs.get_run(run_id)
|
||||
|
||||
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
|
||||
task = asyncio.create_task(self._recover(runner, run_id))
|
||||
self._tasks[run_id] = task
|
||||
task.add_done_callback(lambda t, rid=run_id: self._tasks.pop(rid, None))
|
||||
return await service.store.runs.get_run(run_id)
|
||||
|
||||
async def _recover(self, runner: Any, run_id: str) -> None:
|
||||
try:
|
||||
await runner.recover(run_id)
|
||||
except Exception:
|
||||
logger.exception("MAF recover failed for run %s", run_id)
|
||||
|
||||
async def cancel(self, run_id: str) -> dict[str, Any]:
|
||||
"""Cancel a running workflow AND cascade to DeerFlow sub-runs.
|
||||
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Shared fact ledger for MAF roundtable runs (Phase 3).
|
||||
|
||||
The ledger is the authoritative record of confirmed facts, tool outcomes, and
|
||||
external-action status. Chat text must never be treated as proof that a write
|
||||
succeeded — hosts synthesize conclusions from this structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
_TOOL_STATUSES = frozenset({"completed", "waiting_approval", "failed", "unknown"})
|
||||
_ACTION_STATUSES = frozenset({"pending", "executed", "failed", "unknown", "rejected"})
|
||||
|
||||
|
||||
def empty_ledger() -> dict[str, Any]:
|
||||
"""Return a fresh ledger with all Phase-3 partitions.
|
||||
|
||||
Phase-3 separation of evidence trust levels:
|
||||
- ``candidate_facts`` — unverified seat claims; the model said it, that
|
||||
does not make it true. Each carries a server-generated ``candidate_id``.
|
||||
- ``confirmed_facts`` — only facts a cross-review confirmed (with evidence
|
||||
and a confirmer) ever land here.
|
||||
- ``tool_results`` — outcomes of tools that actually ran.
|
||||
- ``pending_actions`` / ``executed_actions`` / ``failed_or_unknown_actions``
|
||||
— external writes partitioned by lifecycle state.
|
||||
"""
|
||||
return {
|
||||
"candidate_facts": [],
|
||||
"confirmed_facts": [],
|
||||
"evidence": [],
|
||||
"tool_results": [],
|
||||
"executed_actions": [],
|
||||
"pending_actions": [],
|
||||
"failed_or_unknown_actions": [],
|
||||
"conflicts": [],
|
||||
# Legacy alias kept for old checkpoints / prompts.
|
||||
"proposed_actions": [],
|
||||
}
|
||||
|
||||
|
||||
def normalize_ledger(raw: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Coerce a possibly-partial / legacy ledger into the full shape (in-place when possible)."""
|
||||
if not isinstance(raw, dict):
|
||||
return empty_ledger()
|
||||
base = empty_ledger()
|
||||
for key in base:
|
||||
if key not in raw or not isinstance(raw.get(key), list):
|
||||
raw[key] = []
|
||||
return raw
|
||||
|
||||
|
||||
def candidate_id_map(ledger: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
"""Return ``{candidate_id: candidate_entry}`` for O(1) confirmation lookup."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(c["candidate_id"]): c
|
||||
for c in ledger["candidate_facts"]
|
||||
if isinstance(c, dict) and c.get("candidate_id")
|
||||
}
|
||||
|
||||
|
||||
def known_evidence_refs(ledger: dict[str, Any]) -> set[str]:
|
||||
"""The set of evidence references the ledger actually has backing for.
|
||||
|
||||
A cross-review may only confirm a candidate using evidence that the ledger
|
||||
already recorded (either in ``evidence`` or as a successful
|
||||
``tool_results`` entry). This is the trust anchor that stops a weak model
|
||||
from inventing evidence references to promote a seat claim.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
refs: set[str] = set()
|
||||
for e in ledger["evidence"]:
|
||||
if isinstance(e, dict):
|
||||
r = e.get("ref") or e.get("text")
|
||||
if r:
|
||||
refs.add(str(r))
|
||||
elif e:
|
||||
refs.add(str(e))
|
||||
for tr in ledger["tool_results"]:
|
||||
# Only successful tool runs count as evidence (a failed tool proves
|
||||
# nothing). Each tool result is addressable as tool:<name>:<turn>.
|
||||
if isinstance(tr, dict) and tr.get("status") == "completed":
|
||||
name = tr.get("tool_name", "")
|
||||
turn = tr.get("turn_index")
|
||||
if name:
|
||||
refs.add(f"tool:{name}:{turn}" if turn is not None else f"tool:{name}")
|
||||
return refs
|
||||
|
||||
|
||||
def merge_seat_output(
|
||||
ledger: dict[str, Any],
|
||||
seat_output: dict[str, Any],
|
||||
*,
|
||||
agent_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge a validated seat structured output into the ledger.
|
||||
|
||||
Phase-3: seat ``findings`` are written to ``candidate_facts`` (unverified
|
||||
claims), NOT to ``confirmed_facts``. Promotion to ``confirmed_facts``
|
||||
happens only via :func:`confirm_candidate_facts` after cross-review.
|
||||
Proposed actions stay in ``proposed_actions`` — they are never promoted to
|
||||
``executed_actions`` here.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
for finding in seat_output.get("findings") or []:
|
||||
text = str(finding).strip()
|
||||
if text:
|
||||
ledger["candidate_facts"].append({
|
||||
"candidate_id": uuid4().hex,
|
||||
"text": text,
|
||||
"agent_id": agent_id,
|
||||
"source": "seat_finding",
|
||||
"confirmed": False,
|
||||
})
|
||||
for ref in seat_output.get("evidence_refs") or []:
|
||||
if isinstance(ref, dict):
|
||||
entry = dict(ref)
|
||||
entry.setdefault("agent_id", agent_id)
|
||||
ledger["evidence"].append(entry)
|
||||
elif ref:
|
||||
ledger["evidence"].append({"ref": str(ref), "agent_id": agent_id})
|
||||
for conflict in seat_output.get("conflicts") or []:
|
||||
if conflict:
|
||||
ledger["conflicts"].append({"text": str(conflict), "agent_id": agent_id})
|
||||
for action in seat_output.get("proposed_actions") or []:
|
||||
if isinstance(action, dict):
|
||||
entry = dict(action)
|
||||
entry.setdefault("agent_id", agent_id)
|
||||
entry.setdefault("status", "proposed")
|
||||
ledger["proposed_actions"].append(entry)
|
||||
elif action:
|
||||
ledger["proposed_actions"].append({"text": str(action), "agent_id": agent_id, "status": "proposed"})
|
||||
# Executed actions from model text are NOT trusted — only ActionCoordinator
|
||||
# / resume may write executed_actions. Ignore seat_output.executed_actions.
|
||||
return ledger
|
||||
|
||||
|
||||
def confirm_candidates(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
confirmed_candidates: list[dict[str, Any]],
|
||||
confirmed_by: str,
|
||||
rejections: list[dict[str, Any]] | None = None,
|
||||
accepted: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Promote candidate facts to ``confirmed_facts`` after cross-review.
|
||||
|
||||
Phase-3 P1-1 — server-side evidence binding. A candidate is promoted ONLY
|
||||
when **all** of the following hold:
|
||||
|
||||
1. ``candidate_id`` exists in ``candidate_facts`` (the model cannot invent a
|
||||
confirmation for a fact a seat never asserted);
|
||||
2. the candidate carries **at least one** evidence reference;
|
||||
3. every evidence reference resolves to an entry the ledger actually
|
||||
recorded — either in ``evidence`` or as a **successful** ``tool_results``
|
||||
entry. An evidence ref the model invented (not in the ledger) is a
|
||||
hallucination and blocks promotion.
|
||||
|
||||
Candidates that fail any check are left as candidates (``confirmed=False``)
|
||||
and a rejection reason is appended to ``rejections`` so the cross-review
|
||||
summary can explain why a claim was not trusted. When supplied, ``accepted``
|
||||
receives only the server-validated confirmations, so callers never need to
|
||||
infer success from the model's requested confirmation list. Idempotent: a
|
||||
candidate already present in ``confirmed_facts`` is not re-confirmed.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
if rejections is None:
|
||||
rejections = []
|
||||
by_id = candidate_id_map(ledger)
|
||||
valid_refs = known_evidence_refs(ledger)
|
||||
existing_ids = {
|
||||
str(f.get("candidate_id"))
|
||||
for f in ledger["confirmed_facts"]
|
||||
if isinstance(f, dict) and f.get("candidate_id")
|
||||
}
|
||||
for item in confirmed_candidates or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
cid = str(item.get("candidate_id", "")).strip()
|
||||
refs = list(item.get("evidence_refs") or [])
|
||||
if not cid:
|
||||
rejections.append({"reason": "missing_candidate_id"})
|
||||
continue
|
||||
cand = by_id.get(cid)
|
||||
if cand is None:
|
||||
rejections.append({"candidate_id": cid, "reason": "unknown_candidate_id"})
|
||||
continue
|
||||
if cid in existing_ids:
|
||||
# Already confirmed — idempotent no-op.
|
||||
continue
|
||||
if not refs:
|
||||
rejections.append({"candidate_id": cid, "reason": "no_evidence"})
|
||||
continue
|
||||
bad_refs = [r for r in refs if str(r) not in valid_refs]
|
||||
if bad_refs:
|
||||
rejections.append({
|
||||
"candidate_id": cid,
|
||||
"reason": "evidence_not_in_ledger",
|
||||
"evidence_refs": bad_refs,
|
||||
})
|
||||
continue
|
||||
cand["confirmed"] = True
|
||||
ledger["confirmed_facts"].append({
|
||||
"candidate_id": cid,
|
||||
"text": cand.get("text", ""),
|
||||
"agent_id": cand.get("agent_id"),
|
||||
"source": "cross_review_confirmed",
|
||||
"confirmed_by": confirmed_by,
|
||||
"evidence_refs": list(refs),
|
||||
})
|
||||
if accepted is not None:
|
||||
accepted.append({
|
||||
"candidate_id": cid,
|
||||
"evidence_refs": list(refs),
|
||||
})
|
||||
existing_ids.add(cid)
|
||||
return ledger
|
||||
|
||||
|
||||
def confirm_candidate_facts(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
confirmed_candidate_ids: list[str],
|
||||
confirmed_by: str,
|
||||
evidence_refs: list[str] | None = None,
|
||||
accepted: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Back-compat wrapper: confirm a flat id list against one shared evidence set.
|
||||
|
||||
Phase-3 P1-1 tightened confirmation so each candidate must carry at least
|
||||
one evidence reference that the ledger has backing for. The structured
|
||||
path (:func:`confirm_candidates`) is the primary API; this wrapper exists
|
||||
for callers that still pass a flat ``confirmed_candidate_ids`` list — it
|
||||
promotes a candidate ONLY when ``evidence_refs`` is non-empty AND every
|
||||
ref resolves to ledger evidence / successful tool results.
|
||||
"""
|
||||
shared_refs = list(evidence_refs or [])
|
||||
candidates = [
|
||||
{"candidate_id": cid, "evidence_refs": shared_refs}
|
||||
for cid in confirmed_candidate_ids or []
|
||||
]
|
||||
return confirm_candidates(
|
||||
ledger,
|
||||
confirmed_candidates=candidates,
|
||||
confirmed_by=confirmed_by,
|
||||
accepted=accepted,
|
||||
)
|
||||
|
||||
|
||||
def record_tool_result(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
tool_name: str,
|
||||
status: str,
|
||||
agent_id: str,
|
||||
turn_index: int | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a tool result. ``status`` must be one of the allowed tool statuses."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
st = status if status in _TOOL_STATUSES else "unknown"
|
||||
entry: dict[str, Any] = {
|
||||
"tool_name": tool_name,
|
||||
"status": st,
|
||||
"agent_id": agent_id,
|
||||
}
|
||||
if turn_index is not None:
|
||||
entry["turn_index"] = turn_index
|
||||
if detail:
|
||||
entry["detail"] = detail
|
||||
ledger["tool_results"].append(entry)
|
||||
return ledger
|
||||
|
||||
|
||||
def record_action_transition(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
action_id: str,
|
||||
tool_name: str,
|
||||
status: str,
|
||||
agent_id: str | None = None,
|
||||
external_object_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Move / record an external action under the correct ledger partition.
|
||||
|
||||
Rules:
|
||||
- ``proposed`` never becomes ``executed`` via this helper unless status is
|
||||
explicitly ``executed`` (called only after ToolExecutor success).
|
||||
- ``pending`` entries are removed when transitioning to a terminal status.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
st = status if status in _ACTION_STATUSES else "unknown"
|
||||
entry: dict[str, Any] = {
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"status": st,
|
||||
}
|
||||
if agent_id:
|
||||
entry["agent_id"] = agent_id
|
||||
if external_object_id:
|
||||
entry["external_object_id"] = external_object_id
|
||||
|
||||
# Drop matching pending rows when leaving pending.
|
||||
if st != "pending":
|
||||
ledger["pending_actions"] = [
|
||||
p
|
||||
for p in ledger["pending_actions"]
|
||||
if not (isinstance(p, dict) and p.get("action_id") == action_id)
|
||||
]
|
||||
|
||||
if st == "pending":
|
||||
# Replace existing pending with same action_id.
|
||||
ledger["pending_actions"] = [
|
||||
p
|
||||
for p in ledger["pending_actions"]
|
||||
if not (isinstance(p, dict) and p.get("action_id") == action_id)
|
||||
]
|
||||
ledger["pending_actions"].append(entry)
|
||||
elif st == "executed":
|
||||
ledger["executed_actions"] = [
|
||||
e
|
||||
for e in ledger["executed_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
ledger["executed_actions"].append(entry)
|
||||
# Strip from failed list if previously unknown/failed.
|
||||
ledger["failed_or_unknown_actions"] = [
|
||||
e
|
||||
for e in ledger["failed_or_unknown_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
elif st in ("failed", "unknown", "rejected"):
|
||||
ledger["failed_or_unknown_actions"] = [
|
||||
e
|
||||
for e in ledger["failed_or_unknown_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
ledger["failed_or_unknown_actions"].append(entry)
|
||||
# Never keep executed copy of a failed/unknown action.
|
||||
ledger["executed_actions"] = [
|
||||
e
|
||||
for e in ledger["executed_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
return ledger
|
||||
|
||||
|
||||
def assert_ledger_invariants(ledger: dict[str, Any]) -> None:
|
||||
"""Raise AssertionError if the ledger violates evidence/action constraints."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
executed_ids = {
|
||||
e.get("action_id")
|
||||
for e in ledger["executed_actions"]
|
||||
if isinstance(e, dict) and e.get("action_id")
|
||||
}
|
||||
for prop in ledger["proposed_actions"]:
|
||||
if not isinstance(prop, dict):
|
||||
continue
|
||||
# A proposed-only entry must not also appear as executed with status proposed.
|
||||
if prop.get("status") == "proposed" and prop.get("action_id") in executed_ids:
|
||||
# Same action_id may be proposed then later executed — that is OK if
|
||||
# the executed entry has status executed. Only flag if proposed row
|
||||
# itself claims executed.
|
||||
if prop.get("status") == "executed":
|
||||
raise AssertionError("proposed_actions entry has status=executed")
|
||||
for e in ledger["executed_actions"]:
|
||||
if isinstance(e, dict) and e.get("status") not in (None, "executed"):
|
||||
raise AssertionError(f"executed_actions entry has status={e.get('status')!r}")
|
||||
for tr in ledger["tool_results"]:
|
||||
if isinstance(tr, dict) and tr.get("status") not in _TOOL_STATUSES:
|
||||
raise AssertionError(f"invalid tool_results status={tr.get('status')!r}")
|
||||
|
||||
|
||||
def proposed_action_ids(ledger: dict[str, Any]) -> set[str]:
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(p["action_id"])
|
||||
for p in ledger["proposed_actions"]
|
||||
if isinstance(p, dict) and p.get("action_id")
|
||||
}
|
||||
|
||||
|
||||
def executed_action_ids(ledger: dict[str, Any]) -> set[str]:
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(e["action_id"])
|
||||
for e in ledger["executed_actions"]
|
||||
if isinstance(e, dict) and e.get("action_id")
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Seat turn structured-output schema + one-shot format repair (Phase 3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
RepairRunner = Callable[[str], Awaitable[str]]
|
||||
|
||||
|
||||
class SeatTurnOutput(BaseModel):
|
||||
"""Validated structured output produced by an expert seat turn."""
|
||||
|
||||
summary: str = Field(..., min_length=1, description="Short natural-language summary")
|
||||
findings: list[str] = Field(default_factory=list)
|
||||
evidence_refs: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
proposed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
# Model-claimed executed actions are informational only — the fact ledger
|
||||
# never trusts them; ActionCoordinator is the sole source of truth.
|
||||
executed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
open_questions: list[str] = Field(default_factory=list)
|
||||
conflicts: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ConfirmedCandidate(BaseModel):
|
||||
"""A candidate fact the cross-review confirms, WITH required evidence.
|
||||
|
||||
The server only promotes a candidate to confirmed_facts when: the
|
||||
candidate_id exists in candidate_facts AND at least one evidence_ref is
|
||||
supplied AND every evidence_ref resolves to a known evidence/tool result in
|
||||
the ledger. This prevents a weak model from promoting a bare seat claim to
|
||||
an "established fact" with no backing.
|
||||
"""
|
||||
|
||||
candidate_id: str = Field(..., min_length=1)
|
||||
evidence_refs: list[str] = Field(..., min_length=1)
|
||||
|
||||
|
||||
class CrossReviewOutput(BaseModel):
|
||||
"""Validated output of the cross-review stage."""
|
||||
|
||||
summary: str = Field(..., min_length=1)
|
||||
# Conservative default: a review must positively assert evidence is OK; we
|
||||
# never assume it. (Previously defaulted True, which let a model that
|
||||
# omitted the field count as "evidence sufficient".)
|
||||
evidence_ok: bool = False
|
||||
gaps: list[str] = Field(default_factory=list)
|
||||
conflicts: list[str] = Field(default_factory=list)
|
||||
tool_failures: list[str] = Field(default_factory=list)
|
||||
write_status_notes: list[str] = Field(default_factory=list)
|
||||
recommendations: list[str] = Field(default_factory=list)
|
||||
# Phase-3 P1-1: confirmed candidates now carry mandatory evidence. The
|
||||
# legacy bare-id list is still accepted (for back-compat with old prompts)
|
||||
# but is treated as "no evidence" and therefore NOT promoted.
|
||||
confirmed_candidates: list[ConfirmedCandidate] = Field(default_factory=list)
|
||||
confirmed_candidate_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_json_object(text: str) -> str | None:
|
||||
"""Best-effort extract of a JSON object from model text."""
|
||||
raw = (text or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
m = _JSON_FENCE_RE.search(raw)
|
||||
if m:
|
||||
raw = m.group(1).strip()
|
||||
if raw.startswith("{") and raw.endswith("}"):
|
||||
return raw
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
return raw[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
def parse_seat_output(text: str) -> tuple[SeatTurnOutput | None, list[str]]:
|
||||
"""Parse seat text into SeatTurnOutput. Returns (model, readable_errors)."""
|
||||
blob = _extract_json_object(text)
|
||||
if blob is None:
|
||||
return None, ["output is not a JSON object; emit a single JSON object matching SeatTurnOutput"]
|
||||
try:
|
||||
data = json.loads(blob)
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, [f"invalid JSON: {exc.msg}"]
|
||||
if not isinstance(data, dict):
|
||||
return None, ["JSON root must be an object"]
|
||||
try:
|
||||
return SeatTurnOutput.model_validate(data), []
|
||||
except ValidationError as exc:
|
||||
errors = []
|
||||
for err in exc.errors():
|
||||
loc = ".".join(str(p) for p in err.get("loc", ()))
|
||||
errors.append(f"{loc}: {err.get('msg', 'invalid')}")
|
||||
return None, errors or ["schema validation failed"]
|
||||
|
||||
|
||||
def parse_cross_review_output(text: str) -> tuple[CrossReviewOutput | None, list[str]]:
|
||||
blob = _extract_json_object(text)
|
||||
if blob is None:
|
||||
return None, ["output is not a JSON object; emit CrossReviewOutput JSON"]
|
||||
try:
|
||||
data = json.loads(blob)
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, [f"invalid JSON: {exc.msg}"]
|
||||
if not isinstance(data, dict):
|
||||
return None, ["JSON root must be an object"]
|
||||
try:
|
||||
return CrossReviewOutput.model_validate(data), []
|
||||
except ValidationError as exc:
|
||||
errors = [
|
||||
f"{'.'.join(str(p) for p in e.get('loc', ()))}: {e.get('msg', 'invalid')}" for e in exc.errors()
|
||||
]
|
||||
return None, errors or ["schema validation failed"]
|
||||
|
||||
|
||||
def build_repair_prompt(raw: str, errors: list[str], *, schema_name: str = "SeatTurnOutput") -> str:
|
||||
"""Build a one-shot repair instruction for the seat."""
|
||||
err_lines = "\n".join(f"- {e}" for e in errors[:12])
|
||||
return (
|
||||
f"你上一次输出未通过 {schema_name} 校验。请只输出一个 JSON 对象(不要 Markdown 说明),"
|
||||
f"修正下列问题后重试一次:\n{err_lines}\n\n"
|
||||
f"原始输出(供参考,请重写为合法 JSON):\n{raw[:4000]}"
|
||||
)
|
||||
|
||||
|
||||
def seat_output_instruction() -> str:
|
||||
"""Stage instruction fragment telling the model to emit SeatTurnOutput JSON."""
|
||||
return (
|
||||
"请以 JSON 对象回复,字段:summary(string,必填)、findings(string[])、"
|
||||
"evidence_refs、proposed_actions、executed_actions、open_questions、conflicts。"
|
||||
"proposed_actions 只表示建议;不要把未真正由系统执行的外部写入标为已执行。"
|
||||
)
|
||||
|
||||
|
||||
async def validate_or_repair_seat_output(
|
||||
*,
|
||||
raw_content: str,
|
||||
repair_runner: RepairRunner | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate seat output; optionally run one repair turn via ``repair_runner``.
|
||||
|
||||
``repair_runner`` is an async callable ``(repair_prompt: str) -> str``.
|
||||
|
||||
Returns::
|
||||
{
|
||||
"schema_status": "ok" | "schema_invalid" | "repaired",
|
||||
"output": dict | None,
|
||||
"errors": [...],
|
||||
"raw_content": str,
|
||||
"repair_attempted": bool,
|
||||
}
|
||||
"""
|
||||
model, errors = parse_seat_output(raw_content)
|
||||
if model is not None:
|
||||
return {
|
||||
"schema_status": "ok",
|
||||
"output": model.model_dump(),
|
||||
"errors": [],
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": False,
|
||||
}
|
||||
|
||||
if repair_runner is None:
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors,
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": False,
|
||||
}
|
||||
|
||||
repair_prompt = build_repair_prompt(raw_content, errors)
|
||||
try:
|
||||
repaired_text = await repair_runner(repair_prompt)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors + [f"repair failed: {type(exc).__name__}"],
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": True,
|
||||
}
|
||||
|
||||
model2, errors2 = parse_seat_output(str(repaired_text or ""))
|
||||
if model2 is not None:
|
||||
return {
|
||||
"schema_status": "repaired",
|
||||
"output": model2.model_dump(),
|
||||
"errors": [],
|
||||
"raw_content": str(repaired_text),
|
||||
"repair_attempted": True,
|
||||
}
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors2 or errors,
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": True,
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,7 +5,8 @@ The SDK is imported lazily so the Gateway starts without it when the feature
|
||||
flag is off.
|
||||
|
||||
Two topologies:
|
||||
- ``structured``: coordinator → parallel seats → approval gate → finalizer
|
||||
- ``structured`` (Phase 3): planner → parallel experts → cross-review →
|
||||
approval gate → finalizer
|
||||
- ``group_chat``: coordinator loop (dynamic speaker selection) → finalizer
|
||||
|
||||
Each node is a real MAF ``Executor`` subclass instance (from
|
||||
@@ -24,40 +25,42 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def build_structured_workflow(
|
||||
*,
|
||||
coordinator: Any,
|
||||
planner: Any,
|
||||
seat_executors: list[Any],
|
||||
cross_review: Any,
|
||||
approval_gate: Any,
|
||||
finalizer: Any,
|
||||
waiting_terminal: Any,
|
||||
) -> Any:
|
||||
"""Build a structured (fixed-topology) MAF Workflow.
|
||||
|
||||
Topology: coordinator → parallel seat executors → approval gate → finalizer.
|
||||
The approval gate conditionally routes to waiting_terminal (pause) or finalizer.
|
||||
Topology: planner → parallel seat executors → cross_review → approval gate
|
||||
→ finalizer | waiting_terminal.
|
||||
"""
|
||||
from agent_framework import Case, Default, WorkflowBuilder
|
||||
|
||||
builder = WorkflowBuilder(
|
||||
start_executor=coordinator,
|
||||
start_executor=planner,
|
||||
name="maf-structured-roundtable",
|
||||
output_from=[finalizer, waiting_terminal],
|
||||
)
|
||||
|
||||
# coordinator → seats → approval gate. Handle 0/1/≥2 seats since the SDK's
|
||||
# FanOut/FanIn edge groups require at least two members.
|
||||
if len(seat_executors) >= 2:
|
||||
builder.add_fan_out_edges(coordinator, seat_executors)
|
||||
builder.add_fan_in_edges(seat_executors, approval_gate)
|
||||
builder.add_fan_out_edges(planner, seat_executors)
|
||||
builder.add_fan_in_edges(seat_executors, cross_review)
|
||||
elif len(seat_executors) == 1:
|
||||
builder.add_edge(coordinator, seat_executors[0])
|
||||
builder.add_edge(seat_executors[0], approval_gate)
|
||||
builder.add_edge(planner, seat_executors[0])
|
||||
builder.add_edge(seat_executors[0], cross_review)
|
||||
else:
|
||||
# No seats — coordinator feeds the approval gate directly.
|
||||
builder.add_edge(coordinator, approval_gate)
|
||||
builder.add_edge(planner, cross_review)
|
||||
|
||||
builder.add_edge(cross_review, approval_gate)
|
||||
|
||||
# approval gate → conditional: approved → finalizer, waiting → waiting_terminal
|
||||
builder.add_switch_case_edge_group(approval_gate, [
|
||||
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
|
||||
Case(
|
||||
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
|
||||
target=waiting_terminal,
|
||||
),
|
||||
Default(target=finalizer),
|
||||
])
|
||||
|
||||
@@ -86,24 +89,23 @@ def build_group_chat_workflow(
|
||||
output_from=[finalizer, waiting_terminal],
|
||||
)
|
||||
|
||||
# coordinator → switch-case: route to the selected seat, or to approval gate
|
||||
cases = []
|
||||
for i, agent_id in enumerate(seat_agent_ids):
|
||||
cases.append(Case(
|
||||
condition=lambda out, aid=agent_id: isinstance(out, dict) and out.get("agent_id") == aid,
|
||||
target=seat_executors[i],
|
||||
))
|
||||
# Default: coordinator said "done" → go to approval gate
|
||||
cases.append(Default(target=approval_gate))
|
||||
builder.add_switch_case_edge_group(coordinator, cases)
|
||||
|
||||
# each seat → back to coordinator (loop)
|
||||
for seat_exec in seat_executors:
|
||||
builder.add_edge(seat_exec, coordinator)
|
||||
|
||||
# approval gate → conditional: waiting → waiting_terminal, else → finalizer
|
||||
builder.add_switch_case_edge_group(approval_gate, [
|
||||
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
|
||||
Case(
|
||||
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
|
||||
target=waiting_terminal,
|
||||
),
|
||||
Default(target=finalizer),
|
||||
])
|
||||
|
||||
@@ -113,15 +115,19 @@ def build_group_chat_workflow(
|
||||
def create_workflow(
|
||||
*,
|
||||
mode: str,
|
||||
coordinator: Any,
|
||||
seat_executors: list[Any],
|
||||
approval_gate: Any,
|
||||
finalizer: Any,
|
||||
waiting_terminal: Any,
|
||||
seat_agent_ids: list[str],
|
||||
coordinator: Any | None = None,
|
||||
planner: Any | None = None,
|
||||
cross_review: Any | None = None,
|
||||
) -> Any:
|
||||
"""Factory: create the appropriate workflow for the given mode."""
|
||||
if mode == "group_chat":
|
||||
if coordinator is None:
|
||||
raise ValueError("group_chat mode requires coordinator")
|
||||
return build_group_chat_workflow(
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
@@ -130,9 +136,12 @@ def create_workflow(
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=seat_agent_ids,
|
||||
)
|
||||
if planner is None or cross_review is None:
|
||||
raise ValueError("structured mode requires planner and cross_review")
|
||||
return build_structured_workflow(
|
||||
coordinator=coordinator,
|
||||
planner=planner,
|
||||
seat_executors=seat_executors,
|
||||
cross_review=cross_review,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
|
||||
@@ -23,12 +23,15 @@ from typing import Any
|
||||
|
||||
from app.maf_roundtable.action_coordinator import ActionCoordinator
|
||||
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
|
||||
from app.maf_roundtable.fact_ledger import record_action_transition
|
||||
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
||||
from app.maf_roundtable.sanitize import sanitize
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CoordinatorExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
WaitingTerminalExecutor,
|
||||
@@ -92,6 +95,7 @@ class WorkflowRunner:
|
||||
|
||||
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
|
||||
manager_id = session.get("manager_agent_id")
|
||||
seat_timeout, schema_repair = self._orchestration_settings()
|
||||
|
||||
state = RoundtableState(
|
||||
session_id=session_id,
|
||||
@@ -106,28 +110,11 @@ class WorkflowRunner:
|
||||
seat_agent_ids=seat_agents,
|
||||
thread_map=thread_map,
|
||||
capability_snapshot=session.get("capability_snapshot"),
|
||||
seat_timeout_seconds=seat_timeout,
|
||||
schema_repair_enabled=schema_repair,
|
||||
)
|
||||
|
||||
# Build executor instances — one per seat, each bound to a fixed agent_id.
|
||||
coordinator = CoordinatorExecutor(self._adapter, state)
|
||||
seat_executors = [
|
||||
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
||||
for agent_id in seat_agents
|
||||
]
|
||||
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
||||
finalizer = FinalizerExecutor(self._adapter, state)
|
||||
waiting_terminal = WaitingTerminalExecutor()
|
||||
|
||||
workflow = create_workflow(
|
||||
mode=state.mode,
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=seat_agents,
|
||||
)
|
||||
|
||||
workflow = self._build_workflow(state)
|
||||
await self._run_workflow(workflow, state)
|
||||
|
||||
# Persist final state. waiting_approval is NOT a terminal state —
|
||||
@@ -162,35 +149,103 @@ class WorkflowRunner:
|
||||
except Exception as exc:
|
||||
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
|
||||
|
||||
def _orchestration_settings(self) -> tuple[float, bool]:
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
cfg = get_app_config().maf_roundtable
|
||||
return (
|
||||
float(getattr(cfg, "seat_timeout_seconds", 600) or 600),
|
||||
bool(getattr(cfg, "schema_repair_enabled", True)),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return 600.0, True
|
||||
|
||||
def _build_workflow(self, state: RoundtableState) -> Any:
|
||||
seat_executors = [
|
||||
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
||||
for agent_id in state.seat_agent_ids
|
||||
]
|
||||
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
||||
finalizer = FinalizerExecutor(self._adapter, state)
|
||||
waiting_terminal = WaitingTerminalExecutor()
|
||||
|
||||
if state.mode == "group_chat":
|
||||
coordinator = CoordinatorExecutor(self._adapter, state)
|
||||
return create_workflow(
|
||||
mode=state.mode,
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=state.seat_agent_ids,
|
||||
)
|
||||
|
||||
planner = PlannerExecutor(self._adapter, state)
|
||||
cross_review = CrossReviewExecutor(self._adapter, state)
|
||||
return create_workflow(
|
||||
mode="structured",
|
||||
planner=planner,
|
||||
cross_review=cross_review,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=state.seat_agent_ids,
|
||||
)
|
||||
|
||||
async def _persist_checkpoint(self, state: RoundtableState, *, reason: str = "") -> None:
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
||||
|
||||
await self._adapter._emit_critical(
|
||||
state.run_id,
|
||||
make_workflow_checkpointed(
|
||||
None,
|
||||
state.current_round,
|
||||
checkpoint_id=f"nodes-{len(state.completed_nodes)}",
|
||||
reason=reason or f"stage={state.stage}",
|
||||
),
|
||||
)
|
||||
|
||||
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
|
||||
"""Execute the workflow graph via the MAF SDK's streaming API.
|
||||
|
||||
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
|
||||
events. Persists the RoundtableState checkpoint after each superstep.
|
||||
"""
|
||||
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
||||
|
||||
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
|
||||
event_type = getattr(event, "type", "")
|
||||
executor_id = getattr(event, "executor_id", None)
|
||||
|
||||
if event_type == "superstep_completed":
|
||||
# Persist checkpoint after each superstep.
|
||||
iteration = getattr(event, "iteration", None)
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
|
||||
executor_id, state.current_round,
|
||||
checkpoint_id=f"superstep-{iteration}",
|
||||
reason="superstep_completed",
|
||||
))
|
||||
await self._persist_checkpoint(
|
||||
state, reason=f"superstep_completed:{iteration}"
|
||||
)
|
||||
|
||||
elif event_type in ("executor_completed", "executor_invoked"):
|
||||
# Persist after each node so completed_nodes survive restarts.
|
||||
if state.completed_nodes:
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
|
||||
elif event_type == "executor_failed":
|
||||
details = getattr(event, "details", None)
|
||||
error_msg = str(details) if details else "executor failed"
|
||||
# Soft seat failures are handled inside SeatExecutor; only hard
|
||||
# failures (e.g. WriteGuardConfigurationError) abort the run.
|
||||
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
|
||||
|
||||
# Always persist latest completed_nodes at end of stream.
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
|
||||
# Check if the workflow ended in a waiting_approval state.
|
||||
if state.status == "waiting_approval" or state.pending_approvals:
|
||||
state.status = "waiting_approval"
|
||||
@@ -286,10 +341,22 @@ class WorkflowRunner:
|
||||
for pa in state.pending_approvals:
|
||||
approval_id = pa.get("approval_id", "")
|
||||
action_id = pa.get("action_id", "")
|
||||
# Extracted up-front so EVERY branch (rejected / failed / approved)
|
||||
# can write the ledger transition with the correct agent_id.
|
||||
agent_id = pa.get("agent_id", "")
|
||||
thread_id = state.thread_map.get(agent_id, "")
|
||||
tool_name = pa.get("tool_name", "")
|
||||
approval = await self._service.store.approvals.get_approval(approval_id)
|
||||
if approval is None:
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": "approval not found"})
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="failed",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
approval_status = approval.get("status", "pending")
|
||||
@@ -307,17 +374,32 @@ class WorkflowRunner:
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
action_outcomes.append({"action_id": action_id, "status": "rejected"})
|
||||
# Phase-3 P1-3: move the ledger row out of pending into the
|
||||
# failed/unknown partition as rejected.
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="rejected",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if approval_status != "approved":
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": f"unexpected approval status: {approval_status}"})
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="failed",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Approved — execute deterministically via ToolExecutor.
|
||||
agent_id = pa.get("agent_id", "")
|
||||
thread_id = state.thread_map.get(agent_id, "")
|
||||
tool_name = pa.get("tool_name", "")
|
||||
# (agent_id / thread_id / tool_name are already extracted at the
|
||||
# top of this loop iteration so every branch can use them.)
|
||||
|
||||
# Conditional transition (waiting_approval → approved). The
|
||||
# authoritative at-most-once gate is the ToolExecutor's
|
||||
@@ -378,11 +460,20 @@ class WorkflowRunner:
|
||||
"error": outcome.get("error"),
|
||||
})
|
||||
|
||||
state.fact_ledger["pending_actions"].append({
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"outcome": outcome.get("status"),
|
||||
})
|
||||
out_status = outcome.get("status")
|
||||
ledger_status = {
|
||||
"succeeded": "executed",
|
||||
"failed": "failed",
|
||||
"unknown": "unknown",
|
||||
"skipped": "executed",
|
||||
}.get(str(out_status), "unknown")
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=tool_name,
|
||||
status=ledger_status,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
|
||||
# P0-3: determine parent run status from action outcomes.
|
||||
statuses = {o.get("status") for o in action_outcomes}
|
||||
@@ -451,6 +542,76 @@ class WorkflowRunner:
|
||||
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
async def recover(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Graph-level recovery after process restart (MAF-03-07).
|
||||
|
||||
Reloads ``RoundtableState`` from ``maf_checkpoint`` and re-enters the
|
||||
structured workflow. Executors whose ids are in ``completed_nodes``
|
||||
short-circuit without calling DeerFlow again.
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
import agent_framework # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
|
||||
|
||||
run = await self._service.store.runs.get_run(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] not in ("running", "waiting_approval"):
|
||||
logger.info("recover(%s): status=%s — nothing to recover", run_id, run["status"])
|
||||
return run
|
||||
|
||||
checkpoint = run.get("maf_checkpoint")
|
||||
if not checkpoint:
|
||||
logger.warning("recover(%s): no checkpoint", run_id)
|
||||
return run
|
||||
|
||||
state = RoundtableState.from_dict(checkpoint)
|
||||
session = await self._service.store.sessions.get_session(state.session_id, user_id=None)
|
||||
if session is None:
|
||||
return run
|
||||
|
||||
# Ensure threads still bound.
|
||||
participants = await self._service.store.participants.list_participants(state.session_id)
|
||||
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
|
||||
state.thread_map.update(thread_map)
|
||||
|
||||
if run["status"] == "waiting_approval":
|
||||
# Approval path stays on the existing resume() — recover only
|
||||
# continues orchestration for running graphs.
|
||||
return run
|
||||
|
||||
await self._service.store.runs.update_run(run_id, {"status": "running"})
|
||||
try:
|
||||
workflow = self._build_workflow(state)
|
||||
await self._run_workflow(workflow, state)
|
||||
|
||||
latency = int((time.perf_counter() - t0) * 1000)
|
||||
final_status = state.status
|
||||
update_fields: dict[str, Any] = {
|
||||
"status": final_status,
|
||||
"latency_ms": latency,
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
"token_usage": _aggregate_token_usage(state.turn_results),
|
||||
}
|
||||
if final_status in ("completed", "failed", "cancelled"):
|
||||
update_fields["finished_at"] = datetime.now(UTC)
|
||||
await self._service.store.runs.update_run(run_id, update_fields)
|
||||
|
||||
session_fields: dict[str, Any] = {"status": final_status}
|
||||
if final_status == "completed":
|
||||
session_fields["final_result"] = state.final_result
|
||||
if final_status != "waiting_approval":
|
||||
session_fields["fact_ledger"] = state.fact_ledger
|
||||
await self._service.store.sessions.update_session(
|
||||
state.session_id, user_id=self._user_id, fields=session_fields
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
await self._fail_run(run_id, state.session_id, t0, type(exc).__name__, str(exc))
|
||||
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
|
||||
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Aggregate token usage across all seat turns into a run-level summary."""
|
||||
|
||||
Reference in new issue
Block a user