feat(maf): complete resilient roundtable workflow

This commit is contained in:
董宏凯 committed 2026-08-02 00:23:25 +08:00
1 parent bd4131b4e2
commit 21c4c898e7
23 files changed
+4810 -312

No files matched your search

@@ -7,7 +7,8 @@ Phase 1 ships:
- Session CRUD: ``POST /sessions``, ``GET /sessions``, ``GET /sessions/{id}``,
``PUT /sessions/{id}``, ``DELETE /sessions/{id}``.
- Run lifecycle: ``POST /sessions/{id}/runs``, ``GET /runs/{id}``,
``GET /sessions/{id}/runs``, ``POST /runs/{id}/cancel``.
``GET /sessions/{id}/runs``, ``POST /runs/{id}/cancel``,
``POST /runs/{id}/resume``, ``POST /runs/{id}/recover``.
- Event stream: ``GET /runs/{id}/events?after_seq=N`` (断线补拉) — **read-only**;
there is no public ``POST /runs/{id}/events``. Real workflow events are
written only by the backend executor via ``service.append_event``.
@@ -32,7 +33,7 @@ from __future__ import annotations
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi import APIRouter, HTTPException, Query, Request, status
from app.maf_roundtable import (
ActionListResponse,
@@ -291,6 +292,76 @@ async def cancel_run(request: Request, run_id: str) -> RunResponse:
return RunResponse(**final_run)
@router.post("/runs/{run_id}/resume", response_model=RunResponse)
async def resume_run(request: Request, run_id: str) -> RunResponse:
"""Resume after approvals are decided (owner-only).
If the run is not ``waiting_approval`` or still has pending approvals,
returns 409.
"""
svc = _require_service(request)
user_id, is_admin = _user_ctx(request)
try:
run = await svc.get_run(run_id=run_id, user_id=user_id, is_admin=False)
except (MafDisabledError, MafForbiddenError, MafNotFoundError) as exc:
raise _map_service_error(exc) from exc
if run.get("status") != "waiting_approval":
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"run {run_id} is not waiting_approval (status={run.get('status')})",
)
from app.maf_roundtable.dispatcher import get_dispatcher
dispatcher = get_dispatcher(request.app)
try:
updated = await dispatcher.resume(run_id, user_id=user_id or "default")
except Exception as exc: # noqa: BLE001
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"resume failed: {type(exc).__name__}",
) from exc
if updated is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="run not found")
return RunResponse(**updated)
@router.post("/runs/{run_id}/recover", response_model=RunResponse)
async def recover_run(request: Request, run_id: str) -> RunResponse:
"""Graph-level recover after process restart (owner-only).
Re-enters the workflow from checkpoint; completed_nodes are skipped.
Does not silently scan all runs at startup.
"""
svc = _require_service(request)
user_id, is_admin = _user_ctx(request)
try:
run = await svc.get_run(run_id=run_id, user_id=user_id, is_admin=False)
except (MafDisabledError, MafForbiddenError, MafNotFoundError) as exc:
raise _map_service_error(exc) from exc
if run.get("status") not in ("running", "waiting_approval"):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"run {run_id} cannot be recovered (status={run.get('status')})",
)
from app.maf_roundtable.dispatcher import get_dispatcher
dispatcher = get_dispatcher(request.app)
try:
updated = await dispatcher.recover(run_id, user_id=user_id or "default")
except Exception as exc: # noqa: BLE001
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=f"recover failed: {type(exc).__name__}",
) from exc
if updated is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="run not found")
return RunResponse(**updated)
# ── events ─────────────────────────────────────────────────────────────────
@@ -127,11 +127,35 @@ def _format_prior_turns(turns: list[dict[str, Any]], max_per_turn: int) -> str:
def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
parts: list[str] = []
candidates = ledger.get("candidate_facts")
if candidates and isinstance(candidates, list):
# Phase-3: candidate facts are UNVERIFIED seat claims — label them
# explicitly so downstream seats / the finalizer never treat them as
# established conclusions.
parts.append("未经核验的席位主张(不能作为结论):")
for c in candidates[:20]:
if isinstance(c, dict):
parts.append(f" - {c.get('text', c)}")
else:
parts.append(f" - {c}")
facts = ledger.get("confirmed_facts")
if facts and isinstance(facts, list):
parts.append("已确认事实:")
parts.append("已确认事实(经交叉审查核验):")
for f in facts[:20]:
parts.append(f" - {f}")
if isinstance(f, dict):
parts.append(f" - {f.get('text', f)}")
else:
parts.append(f" - {f}")
evidence = ledger.get("evidence")
if evidence and isinstance(evidence, list):
parts.append("证据引用:")
for e in evidence[:10]:
if isinstance(e, dict):
parts.append(f" - {e.get('ref') or e.get('text') or e}")
else:
parts.append(f" - {e}")
tool_results = ledger.get("tool_results")
if tool_results and isinstance(tool_results, list):
@@ -142,6 +166,21 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
else:
parts.append(f" - {tr}")
proposed = ledger.get("proposed_actions")
if proposed and isinstance(proposed, list):
parts.append("建议动作(未执行):")
for p in proposed[:8]:
parts.append(f" - {p}")
executed = ledger.get("executed_actions")
if executed and isinstance(executed, list):
parts.append("已执行外部写入:")
for e in executed[:8]:
if isinstance(e, dict):
parts.append(f" - {e.get('tool_name', '?')} ({e.get('action_id', '')})")
else:
parts.append(f" - {e}")
conflicts = ledger.get("conflicts")
if conflicts and isinstance(conflicts, list):
parts.append("未解决冲突:")
@@ -154,6 +193,12 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
for p in pending[:5]:
parts.append(f" - {p}")
failed = ledger.get("failed_or_unknown_actions")
if failed and isinstance(failed, list):
parts.append("失败或状态不确定的动作:")
for f in failed[:5]:
parts.append(f" - {f}")
result = "\n".join(parts)
if len(result) > max_chars:
result = result[:max_chars]
@@ -13,7 +13,8 @@ Architecture:
- Cancellation propagates from MAF run → DeerFlow run via RunManager.cancel.
- All seat runs persisted to ``maf_roundtable_seat_runs`` (survives restart).
- Critical audit events (tool.*, skill.*, approval.*, artifact.*) MUST persist;
failure stops external writes and marks the run as failed.
audit failure stops external writes; seat turn failures are soft-failed
into ``seat_outcomes`` / ``partial_failure`` (structured mode).
Boundary: lives in ``app.maf_roundtable`` (may import ``deerflow.*`` and
``app.gateway.*``); never imported by ``deerflow.*``.
@@ -132,6 +133,7 @@ class DeerFlowAgentAdapter:
turn_index: int,
message: str,
thread_id: str,
execution_stage: str = "expert",
model_name: str | None = None,
reasoning_effort: str | None = None,
thinking_enabled: bool = True,
@@ -147,6 +149,7 @@ class DeerFlowAgentAdapter:
capability_snapshot: dict[str, Any] | None = None,
write_policy: str = "confirm_high_risk",
approved_tool: str | None = None,
wait_timeout_seconds: float | None = None,
) -> SeatResponse:
"""Execute one seat turn through the DeerFlow Agent Runtime."""
t0 = time.perf_counter()
@@ -166,10 +169,56 @@ class DeerFlowAgentAdapter:
principal = ExecutionPrincipal(user_id=self._user_id, session_id=session_id, run_id=run_id)
seat_run = await self._service.store.seat_runs.create_seat_run(
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index, deerflow_thread_id=thread_id
# Phase-3 P0-1: atomic "occupy or read". Only the caller that wins the
# insert (created=True) may start a DeerFlow run. An existing row is
# handled by status so we NEVER silently start a second DeerFlow
# execution for the same four-tuple (run, agent, turn, stage).
seat_run, created = await self._service.store.seat_runs.get_or_create_seat_run(
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index,
execution_stage=execution_stage, deerflow_thread_id=thread_id,
)
seat_run_id = seat_run["id"]
prior_status = seat_run.get("status")
if not created:
if prior_status == "completed":
return self._seat_response_from_snapshot(
seat_run=seat_run,
agent_id=agent_id,
turn_index=turn_index,
)
if prior_status in ("pending", "running", "cancelling"):
effective_timeout = wait_timeout_seconds if wait_timeout_seconds is not None else 600.0
seat_run = await self._service.store.seat_runs.wait_for_terminal_seat_run(
maf_run_id=run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
timeout_seconds=effective_timeout,
)
if seat_run["status"] == "completed":
return self._seat_response_from_snapshot(
seat_run=seat_run,
agent_id=agent_id,
turn_index=turn_index,
)
return SeatResponse(
agent_id=agent_id,
turn_index=turn_index,
content="",
error=f"stage '{execution_stage}' ended as {seat_run['status']}",
metadata={"seat_run_id": seat_run["id"], "terminal": seat_run["status"]},
latency_ms=int((time.perf_counter() - t0) * 1000),
)
return SeatResponse(
agent_id=agent_id,
turn_index=turn_index,
content="",
error=f"stage '{execution_stage}' previously terminal ({prior_status}); "
f"manual recovery required",
metadata={"terminal": prior_status, "seat_run_id": seat_run_id},
latency_ms=int((time.perf_counter() - t0) * 1000),
)
try:
response = await self._execute_run(
@@ -210,7 +259,16 @@ class DeerFlowAgentAdapter:
latency_ms=latency,
)
await self._service.store.seat_runs.mark_completed(seat_run_id)
raw_snapshot = {
"content": response.content,
"tool_calls": response.tool_calls,
"usage": response.usage,
"metadata": response.metadata,
}
snapshot = sanitize(raw_snapshot)
if not isinstance(snapshot, dict):
raise RuntimeError("seat result snapshot sanitization produced invalid payload")
await self._service.store.seat_runs.mark_completed(seat_run_id, result_snapshot=snapshot)
response.latency_ms = int((time.perf_counter() - t0) * 1000)
await self._emit_critical(run_id, make_turn_completed(
agent_id, turn_index,
@@ -463,6 +521,34 @@ class DeerFlowAgentAdapter:
# ── internal helpers ────────────────────────────────────────────────────
def _seat_response_from_snapshot(
self,
*,
seat_run: dict[str, Any],
agent_id: str,
turn_index: int,
) -> SeatResponse:
snapshot = seat_run.get("result_snapshot")
if not snapshot or not isinstance(snapshot, dict):
return SeatResponse(
agent_id=agent_id,
turn_index=turn_index,
content="",
error="completed seat run has no replayable snapshot; manual recovery required",
metadata={"seat_run_id": seat_run.get("id"), "terminal": seat_run.get("status")},
latency_ms=0,
)
return SeatResponse(
agent_id=agent_id,
turn_index=turn_index,
content=str(snapshot.get("content", "")),
tool_calls=list(snapshot.get("tool_calls") or []),
usage=snapshot.get("usage"),
metadata=dict(snapshot.get("metadata") or {}),
error=None,
latency_ms=0,
)
def _latest_ai_text(self, messages: list[dict[str, Any]]) -> str:
for m in reversed(messages):
if not isinstance(m, dict):
@@ -96,6 +96,36 @@ class MafRunDispatcher:
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
return await runner.resume(run_id)
async def recover(self, run_id: str, *, user_id: str) -> dict[str, Any] | None:
"""Re-enter the workflow from checkpoint after a process restart.
Unlike ``resume`` (approval path), this continues orchestration for
``running`` runs and skips ``completed_nodes``.
"""
from app.maf_roundtable.workflow_runner import WorkflowRunner
service = getattr(self._app.state, "maf_roundtable_service", None)
if service is None:
raise MafSdkNotAvailableError("MAF roundtable service not available")
# Avoid overlapping with an in-process execute task.
existing = self._tasks.get(run_id)
if existing is not None and not existing.done():
logger.info("recover(%s): background task still running — skip", run_id)
return await service.store.runs.get_run(run_id)
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
task = asyncio.create_task(self._recover(runner, run_id))
self._tasks[run_id] = task
task.add_done_callback(lambda t, rid=run_id: self._tasks.pop(rid, None))
return await service.store.runs.get_run(run_id)
async def _recover(self, runner: Any, run_id: str) -> None:
try:
await runner.recover(run_id)
except Exception:
logger.exception("MAF recover failed for run %s", run_id)
async def cancel(self, run_id: str) -> dict[str, Any]:
"""Cancel a running workflow AND cascade to DeerFlow sub-runs.
@@ -0,0 +1,391 @@
"""Shared fact ledger for MAF roundtable runs (Phase 3).
The ledger is the authoritative record of confirmed facts, tool outcomes, and
external-action status. Chat text must never be treated as proof that a write
succeeded — hosts synthesize conclusions from this structure.
"""
from __future__ import annotations
from typing import Any
from uuid import uuid4
_TOOL_STATUSES = frozenset({"completed", "waiting_approval", "failed", "unknown"})
_ACTION_STATUSES = frozenset({"pending", "executed", "failed", "unknown", "rejected"})
def empty_ledger() -> dict[str, Any]:
"""Return a fresh ledger with all Phase-3 partitions.
Phase-3 separation of evidence trust levels:
- ``candidate_facts`` — unverified seat claims; the model said it, that
does not make it true. Each carries a server-generated ``candidate_id``.
- ``confirmed_facts`` — only facts a cross-review confirmed (with evidence
and a confirmer) ever land here.
- ``tool_results`` — outcomes of tools that actually ran.
- ``pending_actions`` / ``executed_actions`` / ``failed_or_unknown_actions``
— external writes partitioned by lifecycle state.
"""
return {
"candidate_facts": [],
"confirmed_facts": [],
"evidence": [],
"tool_results": [],
"executed_actions": [],
"pending_actions": [],
"failed_or_unknown_actions": [],
"conflicts": [],
# Legacy alias kept for old checkpoints / prompts.
"proposed_actions": [],
}
def normalize_ledger(raw: dict[str, Any] | None) -> dict[str, Any]:
"""Coerce a possibly-partial / legacy ledger into the full shape (in-place when possible)."""
if not isinstance(raw, dict):
return empty_ledger()
base = empty_ledger()
for key in base:
if key not in raw or not isinstance(raw.get(key), list):
raw[key] = []
return raw
def candidate_id_map(ledger: dict[str, Any]) -> dict[str, dict[str, Any]]:
"""Return ``{candidate_id: candidate_entry}`` for O(1) confirmation lookup."""
ledger = normalize_ledger(ledger)
return {
str(c["candidate_id"]): c
for c in ledger["candidate_facts"]
if isinstance(c, dict) and c.get("candidate_id")
}
def known_evidence_refs(ledger: dict[str, Any]) -> set[str]:
"""The set of evidence references the ledger actually has backing for.
A cross-review may only confirm a candidate using evidence that the ledger
already recorded (either in ``evidence`` or as a successful
``tool_results`` entry). This is the trust anchor that stops a weak model
from inventing evidence references to promote a seat claim.
"""
ledger = normalize_ledger(ledger)
refs: set[str] = set()
for e in ledger["evidence"]:
if isinstance(e, dict):
r = e.get("ref") or e.get("text")
if r:
refs.add(str(r))
elif e:
refs.add(str(e))
for tr in ledger["tool_results"]:
# Only successful tool runs count as evidence (a failed tool proves
# nothing). Each tool result is addressable as tool:<name>:<turn>.
if isinstance(tr, dict) and tr.get("status") == "completed":
name = tr.get("tool_name", "")
turn = tr.get("turn_index")
if name:
refs.add(f"tool:{name}:{turn}" if turn is not None else f"tool:{name}")
return refs
def merge_seat_output(
ledger: dict[str, Any],
seat_output: dict[str, Any],
*,
agent_id: str,
) -> dict[str, Any]:
"""Merge a validated seat structured output into the ledger.
Phase-3: seat ``findings`` are written to ``candidate_facts`` (unverified
claims), NOT to ``confirmed_facts``. Promotion to ``confirmed_facts``
happens only via :func:`confirm_candidate_facts` after cross-review.
Proposed actions stay in ``proposed_actions`` — they are never promoted to
``executed_actions`` here.
"""
ledger = normalize_ledger(ledger)
for finding in seat_output.get("findings") or []:
text = str(finding).strip()
if text:
ledger["candidate_facts"].append({
"candidate_id": uuid4().hex,
"text": text,
"agent_id": agent_id,
"source": "seat_finding",
"confirmed": False,
})
for ref in seat_output.get("evidence_refs") or []:
if isinstance(ref, dict):
entry = dict(ref)
entry.setdefault("agent_id", agent_id)
ledger["evidence"].append(entry)
elif ref:
ledger["evidence"].append({"ref": str(ref), "agent_id": agent_id})
for conflict in seat_output.get("conflicts") or []:
if conflict:
ledger["conflicts"].append({"text": str(conflict), "agent_id": agent_id})
for action in seat_output.get("proposed_actions") or []:
if isinstance(action, dict):
entry = dict(action)
entry.setdefault("agent_id", agent_id)
entry.setdefault("status", "proposed")
ledger["proposed_actions"].append(entry)
elif action:
ledger["proposed_actions"].append({"text": str(action), "agent_id": agent_id, "status": "proposed"})
# Executed actions from model text are NOT trusted — only ActionCoordinator
# / resume may write executed_actions. Ignore seat_output.executed_actions.
return ledger
def confirm_candidates(
ledger: dict[str, Any],
*,
confirmed_candidates: list[dict[str, Any]],
confirmed_by: str,
rejections: list[dict[str, Any]] | None = None,
accepted: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Promote candidate facts to ``confirmed_facts`` after cross-review.
Phase-3 P1-1 — server-side evidence binding. A candidate is promoted ONLY
when **all** of the following hold:
1. ``candidate_id`` exists in ``candidate_facts`` (the model cannot invent a
confirmation for a fact a seat never asserted);
2. the candidate carries **at least one** evidence reference;
3. every evidence reference resolves to an entry the ledger actually
recorded — either in ``evidence`` or as a **successful** ``tool_results``
entry. An evidence ref the model invented (not in the ledger) is a
hallucination and blocks promotion.
Candidates that fail any check are left as candidates (``confirmed=False``)
and a rejection reason is appended to ``rejections`` so the cross-review
summary can explain why a claim was not trusted. When supplied, ``accepted``
receives only the server-validated confirmations, so callers never need to
infer success from the model's requested confirmation list. Idempotent: a
candidate already present in ``confirmed_facts`` is not re-confirmed.
"""
ledger = normalize_ledger(ledger)
if rejections is None:
rejections = []
by_id = candidate_id_map(ledger)
valid_refs = known_evidence_refs(ledger)
existing_ids = {
str(f.get("candidate_id"))
for f in ledger["confirmed_facts"]
if isinstance(f, dict) and f.get("candidate_id")
}
for item in confirmed_candidates or []:
if not isinstance(item, dict):
continue
cid = str(item.get("candidate_id", "")).strip()
refs = list(item.get("evidence_refs") or [])
if not cid:
rejections.append({"reason": "missing_candidate_id"})
continue
cand = by_id.get(cid)
if cand is None:
rejections.append({"candidate_id": cid, "reason": "unknown_candidate_id"})
continue
if cid in existing_ids:
# Already confirmed — idempotent no-op.
continue
if not refs:
rejections.append({"candidate_id": cid, "reason": "no_evidence"})
continue
bad_refs = [r for r in refs if str(r) not in valid_refs]
if bad_refs:
rejections.append({
"candidate_id": cid,
"reason": "evidence_not_in_ledger",
"evidence_refs": bad_refs,
})
continue
cand["confirmed"] = True
ledger["confirmed_facts"].append({
"candidate_id": cid,
"text": cand.get("text", ""),
"agent_id": cand.get("agent_id"),
"source": "cross_review_confirmed",
"confirmed_by": confirmed_by,
"evidence_refs": list(refs),
})
if accepted is not None:
accepted.append({
"candidate_id": cid,
"evidence_refs": list(refs),
})
existing_ids.add(cid)
return ledger
def confirm_candidate_facts(
ledger: dict[str, Any],
*,
confirmed_candidate_ids: list[str],
confirmed_by: str,
evidence_refs: list[str] | None = None,
accepted: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Back-compat wrapper: confirm a flat id list against one shared evidence set.
Phase-3 P1-1 tightened confirmation so each candidate must carry at least
one evidence reference that the ledger has backing for. The structured
path (:func:`confirm_candidates`) is the primary API; this wrapper exists
for callers that still pass a flat ``confirmed_candidate_ids`` list — it
promotes a candidate ONLY when ``evidence_refs`` is non-empty AND every
ref resolves to ledger evidence / successful tool results.
"""
shared_refs = list(evidence_refs or [])
candidates = [
{"candidate_id": cid, "evidence_refs": shared_refs}
for cid in confirmed_candidate_ids or []
]
return confirm_candidates(
ledger,
confirmed_candidates=candidates,
confirmed_by=confirmed_by,
accepted=accepted,
)
def record_tool_result(
ledger: dict[str, Any],
*,
tool_name: str,
status: str,
agent_id: str,
turn_index: int | None = None,
detail: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Append a tool result. ``status`` must be one of the allowed tool statuses."""
ledger = normalize_ledger(ledger)
st = status if status in _TOOL_STATUSES else "unknown"
entry: dict[str, Any] = {
"tool_name": tool_name,
"status": st,
"agent_id": agent_id,
}
if turn_index is not None:
entry["turn_index"] = turn_index
if detail:
entry["detail"] = detail
ledger["tool_results"].append(entry)
return ledger
def record_action_transition(
ledger: dict[str, Any],
*,
action_id: str,
tool_name: str,
status: str,
agent_id: str | None = None,
external_object_id: str | None = None,
) -> dict[str, Any]:
"""Move / record an external action under the correct ledger partition.
Rules:
- ``proposed`` never becomes ``executed`` via this helper unless status is
explicitly ``executed`` (called only after ToolExecutor success).
- ``pending`` entries are removed when transitioning to a terminal status.
"""
ledger = normalize_ledger(ledger)
st = status if status in _ACTION_STATUSES else "unknown"
entry: dict[str, Any] = {
"action_id": action_id,
"tool_name": tool_name,
"status": st,
}
if agent_id:
entry["agent_id"] = agent_id
if external_object_id:
entry["external_object_id"] = external_object_id
# Drop matching pending rows when leaving pending.
if st != "pending":
ledger["pending_actions"] = [
p
for p in ledger["pending_actions"]
if not (isinstance(p, dict) and p.get("action_id") == action_id)
]
if st == "pending":
# Replace existing pending with same action_id.
ledger["pending_actions"] = [
p
for p in ledger["pending_actions"]
if not (isinstance(p, dict) and p.get("action_id") == action_id)
]
ledger["pending_actions"].append(entry)
elif st == "executed":
ledger["executed_actions"] = [
e
for e in ledger["executed_actions"]
if not (isinstance(e, dict) and e.get("action_id") == action_id)
]
ledger["executed_actions"].append(entry)
# Strip from failed list if previously unknown/failed.
ledger["failed_or_unknown_actions"] = [
e
for e in ledger["failed_or_unknown_actions"]
if not (isinstance(e, dict) and e.get("action_id") == action_id)
]
elif st in ("failed", "unknown", "rejected"):
ledger["failed_or_unknown_actions"] = [
e
for e in ledger["failed_or_unknown_actions"]
if not (isinstance(e, dict) and e.get("action_id") == action_id)
]
ledger["failed_or_unknown_actions"].append(entry)
# Never keep executed copy of a failed/unknown action.
ledger["executed_actions"] = [
e
for e in ledger["executed_actions"]
if not (isinstance(e, dict) and e.get("action_id") == action_id)
]
return ledger
def assert_ledger_invariants(ledger: dict[str, Any]) -> None:
"""Raise AssertionError if the ledger violates evidence/action constraints."""
ledger = normalize_ledger(ledger)
executed_ids = {
e.get("action_id")
for e in ledger["executed_actions"]
if isinstance(e, dict) and e.get("action_id")
}
for prop in ledger["proposed_actions"]:
if not isinstance(prop, dict):
continue
# A proposed-only entry must not also appear as executed with status proposed.
if prop.get("status") == "proposed" and prop.get("action_id") in executed_ids:
# Same action_id may be proposed then later executed — that is OK if
# the executed entry has status executed. Only flag if proposed row
# itself claims executed.
if prop.get("status") == "executed":
raise AssertionError("proposed_actions entry has status=executed")
for e in ledger["executed_actions"]:
if isinstance(e, dict) and e.get("status") not in (None, "executed"):
raise AssertionError(f"executed_actions entry has status={e.get('status')!r}")
for tr in ledger["tool_results"]:
if isinstance(tr, dict) and tr.get("status") not in _TOOL_STATUSES:
raise AssertionError(f"invalid tool_results status={tr.get('status')!r}")
def proposed_action_ids(ledger: dict[str, Any]) -> set[str]:
ledger = normalize_ledger(ledger)
return {
str(p["action_id"])
for p in ledger["proposed_actions"]
if isinstance(p, dict) and p.get("action_id")
}
def executed_action_ids(ledger: dict[str, Any]) -> set[str]:
ledger = normalize_ledger(ledger)
return {
str(e["action_id"])
for e in ledger["executed_actions"]
if isinstance(e, dict) and e.get("action_id")
}
@@ -0,0 +1,206 @@
"""Seat turn structured-output schema + one-shot format repair (Phase 3)."""
from __future__ import annotations
import json
import re
from collections.abc import Awaitable, Callable
from typing import Any
from pydantic import BaseModel, Field, ValidationError
RepairRunner = Callable[[str], Awaitable[str]]
class SeatTurnOutput(BaseModel):
"""Validated structured output produced by an expert seat turn."""
summary: str = Field(..., min_length=1, description="Short natural-language summary")
findings: list[str] = Field(default_factory=list)
evidence_refs: list[str | dict[str, Any]] = Field(default_factory=list)
proposed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
# Model-claimed executed actions are informational only — the fact ledger
# never trusts them; ActionCoordinator is the sole source of truth.
executed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
open_questions: list[str] = Field(default_factory=list)
conflicts: list[str] = Field(default_factory=list)
class ConfirmedCandidate(BaseModel):
"""A candidate fact the cross-review confirms, WITH required evidence.
The server only promotes a candidate to confirmed_facts when: the
candidate_id exists in candidate_facts AND at least one evidence_ref is
supplied AND every evidence_ref resolves to a known evidence/tool result in
the ledger. This prevents a weak model from promoting a bare seat claim to
an "established fact" with no backing.
"""
candidate_id: str = Field(..., min_length=1)
evidence_refs: list[str] = Field(..., min_length=1)
class CrossReviewOutput(BaseModel):
"""Validated output of the cross-review stage."""
summary: str = Field(..., min_length=1)
# Conservative default: a review must positively assert evidence is OK; we
# never assume it. (Previously defaulted True, which let a model that
# omitted the field count as "evidence sufficient".)
evidence_ok: bool = False
gaps: list[str] = Field(default_factory=list)
conflicts: list[str] = Field(default_factory=list)
tool_failures: list[str] = Field(default_factory=list)
write_status_notes: list[str] = Field(default_factory=list)
recommendations: list[str] = Field(default_factory=list)
# Phase-3 P1-1: confirmed candidates now carry mandatory evidence. The
# legacy bare-id list is still accepted (for back-compat with old prompts)
# but is treated as "no evidence" and therefore NOT promoted.
confirmed_candidates: list[ConfirmedCandidate] = Field(default_factory=list)
confirmed_candidate_ids: list[str] = Field(default_factory=list)
_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
def _extract_json_object(text: str) -> str | None:
"""Best-effort extract of a JSON object from model text."""
raw = (text or "").strip()
if not raw:
return None
m = _JSON_FENCE_RE.search(raw)
if m:
raw = m.group(1).strip()
if raw.startswith("{") and raw.endswith("}"):
return raw
start = raw.find("{")
end = raw.rfind("}")
if start >= 0 and end > start:
return raw[start : end + 1]
return None
def parse_seat_output(text: str) -> tuple[SeatTurnOutput | None, list[str]]:
"""Parse seat text into SeatTurnOutput. Returns (model, readable_errors)."""
blob = _extract_json_object(text)
if blob is None:
return None, ["output is not a JSON object; emit a single JSON object matching SeatTurnOutput"]
try:
data = json.loads(blob)
except json.JSONDecodeError as exc:
return None, [f"invalid JSON: {exc.msg}"]
if not isinstance(data, dict):
return None, ["JSON root must be an object"]
try:
return SeatTurnOutput.model_validate(data), []
except ValidationError as exc:
errors = []
for err in exc.errors():
loc = ".".join(str(p) for p in err.get("loc", ()))
errors.append(f"{loc}: {err.get('msg', 'invalid')}")
return None, errors or ["schema validation failed"]
def parse_cross_review_output(text: str) -> tuple[CrossReviewOutput | None, list[str]]:
blob = _extract_json_object(text)
if blob is None:
return None, ["output is not a JSON object; emit CrossReviewOutput JSON"]
try:
data = json.loads(blob)
except json.JSONDecodeError as exc:
return None, [f"invalid JSON: {exc.msg}"]
if not isinstance(data, dict):
return None, ["JSON root must be an object"]
try:
return CrossReviewOutput.model_validate(data), []
except ValidationError as exc:
errors = [
f"{'.'.join(str(p) for p in e.get('loc', ()))}: {e.get('msg', 'invalid')}" for e in exc.errors()
]
return None, errors or ["schema validation failed"]
def build_repair_prompt(raw: str, errors: list[str], *, schema_name: str = "SeatTurnOutput") -> str:
"""Build a one-shot repair instruction for the seat."""
err_lines = "\n".join(f"- {e}" for e in errors[:12])
return (
f"你上一次输出未通过 {schema_name} 校验。请只输出一个 JSON 对象(不要 Markdown 说明),"
f"修正下列问题后重试一次:\n{err_lines}\n\n"
f"原始输出(供参考,请重写为合法 JSON):\n{raw[:4000]}"
)
def seat_output_instruction() -> str:
"""Stage instruction fragment telling the model to emit SeatTurnOutput JSON."""
return (
"请以 JSON 对象回复,字段:summary(string,必填)、findings(string[])、"
"evidence_refs、proposed_actions、executed_actions、open_questions、conflicts。"
"proposed_actions 只表示建议;不要把未真正由系统执行的外部写入标为已执行。"
)
async def validate_or_repair_seat_output(
*,
raw_content: str,
repair_runner: RepairRunner | None,
) -> dict[str, Any]:
"""Validate seat output; optionally run one repair turn via ``repair_runner``.
``repair_runner`` is an async callable ``(repair_prompt: str) -> str``.
Returns::
{
"schema_status": "ok" | "schema_invalid" | "repaired",
"output": dict | None,
"errors": [...],
"raw_content": str,
"repair_attempted": bool,
}
"""
model, errors = parse_seat_output(raw_content)
if model is not None:
return {
"schema_status": "ok",
"output": model.model_dump(),
"errors": [],
"raw_content": raw_content,
"repair_attempted": False,
}
if repair_runner is None:
return {
"schema_status": "schema_invalid",
"output": None,
"errors": errors,
"raw_content": raw_content,
"repair_attempted": False,
}
repair_prompt = build_repair_prompt(raw_content, errors)
try:
repaired_text = await repair_runner(repair_prompt)
except Exception as exc: # noqa: BLE001
return {
"schema_status": "schema_invalid",
"output": None,
"errors": errors + [f"repair failed: {type(exc).__name__}"],
"raw_content": raw_content,
"repair_attempted": True,
}
model2, errors2 = parse_seat_output(str(repaired_text or ""))
if model2 is not None:
return {
"schema_status": "repaired",
"output": model2.model_dump(),
"errors": [],
"raw_content": str(repaired_text),
"repair_attempted": True,
}
return {
"schema_status": "schema_invalid",
"output": None,
"errors": errors2 or errors,
"raw_content": raw_content,
"repair_attempted": True,
}
File diff suppressed because it is too large. Load diff
@@ -5,7 +5,8 @@ The SDK is imported lazily so the Gateway starts without it when the feature
flag is off.
Two topologies:
- ``structured``: coordinator → parallel seats → approval gate → finalizer
- ``structured`` (Phase 3): planner → parallel experts → cross-review →
approval gate → finalizer
- ``group_chat``: coordinator loop (dynamic speaker selection) → finalizer
Each node is a real MAF ``Executor`` subclass instance (from
@@ -24,40 +25,42 @@ logger = logging.getLogger(__name__)
def build_structured_workflow(
*,
coordinator: Any,
planner: Any,
seat_executors: list[Any],
cross_review: Any,
approval_gate: Any,
finalizer: Any,
waiting_terminal: Any,
) -> Any:
"""Build a structured (fixed-topology) MAF Workflow.
Topology: coordinator → parallel seat executors → approval gate → finalizer.
The approval gate conditionally routes to waiting_terminal (pause) or finalizer.
Topology: planner → parallel seat executors → cross_review → approval gate
→ finalizer | waiting_terminal.
"""
from agent_framework import Case, Default, WorkflowBuilder
builder = WorkflowBuilder(
start_executor=coordinator,
start_executor=planner,
name="maf-structured-roundtable",
output_from=[finalizer, waiting_terminal],
)
# coordinator → seats → approval gate. Handle 0/1/≥2 seats since the SDK's
# FanOut/FanIn edge groups require at least two members.
if len(seat_executors) >= 2:
builder.add_fan_out_edges(coordinator, seat_executors)
builder.add_fan_in_edges(seat_executors, approval_gate)
builder.add_fan_out_edges(planner, seat_executors)
builder.add_fan_in_edges(seat_executors, cross_review)
elif len(seat_executors) == 1:
builder.add_edge(coordinator, seat_executors[0])
builder.add_edge(seat_executors[0], approval_gate)
builder.add_edge(planner, seat_executors[0])
builder.add_edge(seat_executors[0], cross_review)
else:
# No seats — coordinator feeds the approval gate directly.
builder.add_edge(coordinator, approval_gate)
builder.add_edge(planner, cross_review)
builder.add_edge(cross_review, approval_gate)
# approval gate → conditional: approved → finalizer, waiting → waiting_terminal
builder.add_switch_case_edge_group(approval_gate, [
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
Case(
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
target=waiting_terminal,
),
Default(target=finalizer),
])
@@ -86,24 +89,23 @@ def build_group_chat_workflow(
output_from=[finalizer, waiting_terminal],
)
# coordinator → switch-case: route to the selected seat, or to approval gate
cases = []
for i, agent_id in enumerate(seat_agent_ids):
cases.append(Case(
condition=lambda out, aid=agent_id: isinstance(out, dict) and out.get("agent_id") == aid,
target=seat_executors[i],
))
# Default: coordinator said "done" → go to approval gate
cases.append(Default(target=approval_gate))
builder.add_switch_case_edge_group(coordinator, cases)
# each seat → back to coordinator (loop)
for seat_exec in seat_executors:
builder.add_edge(seat_exec, coordinator)
# approval gate → conditional: waiting → waiting_terminal, else → finalizer
builder.add_switch_case_edge_group(approval_gate, [
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
Case(
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
target=waiting_terminal,
),
Default(target=finalizer),
])
@@ -113,15 +115,19 @@ def build_group_chat_workflow(
def create_workflow(
*,
mode: str,
coordinator: Any,
seat_executors: list[Any],
approval_gate: Any,
finalizer: Any,
waiting_terminal: Any,
seat_agent_ids: list[str],
coordinator: Any | None = None,
planner: Any | None = None,
cross_review: Any | None = None,
) -> Any:
"""Factory: create the appropriate workflow for the given mode."""
if mode == "group_chat":
if coordinator is None:
raise ValueError("group_chat mode requires coordinator")
return build_group_chat_workflow(
coordinator=coordinator,
seat_executors=seat_executors,
@@ -130,9 +136,12 @@ def create_workflow(
waiting_terminal=waiting_terminal,
seat_agent_ids=seat_agent_ids,
)
if planner is None or cross_review is None:
raise ValueError("structured mode requires planner and cross_review")
return build_structured_workflow(
coordinator=coordinator,
planner=planner,
seat_executors=seat_executors,
cross_review=cross_review,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
@@ -23,12 +23,15 @@ from typing import Any
from app.maf_roundtable.action_coordinator import ActionCoordinator
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
from app.maf_roundtable.fact_ledger import record_action_transition
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
from app.maf_roundtable.sanitize import sanitize
from app.maf_roundtable.workflow_executors import (
ApprovalGateExecutor,
CoordinatorExecutor,
CrossReviewExecutor,
FinalizerExecutor,
PlannerExecutor,
RoundtableState,
SeatExecutor,
WaitingTerminalExecutor,
@@ -92,6 +95,7 @@ class WorkflowRunner:
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
manager_id = session.get("manager_agent_id")
seat_timeout, schema_repair = self._orchestration_settings()
state = RoundtableState(
session_id=session_id,
@@ -106,28 +110,11 @@ class WorkflowRunner:
seat_agent_ids=seat_agents,
thread_map=thread_map,
capability_snapshot=session.get("capability_snapshot"),
seat_timeout_seconds=seat_timeout,
schema_repair_enabled=schema_repair,
)
# Build executor instances — one per seat, each bound to a fixed agent_id.
coordinator = CoordinatorExecutor(self._adapter, state)
seat_executors = [
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
for agent_id in seat_agents
]
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
finalizer = FinalizerExecutor(self._adapter, state)
waiting_terminal = WaitingTerminalExecutor()
workflow = create_workflow(
mode=state.mode,
coordinator=coordinator,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=seat_agents,
)
workflow = self._build_workflow(state)
await self._run_workflow(workflow, state)
# Persist final state. waiting_approval is NOT a terminal state —
@@ -162,35 +149,103 @@ class WorkflowRunner:
except Exception as exc:
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
def _orchestration_settings(self) -> tuple[float, bool]:
try:
from deerflow.config import get_app_config
cfg = get_app_config().maf_roundtable
return (
float(getattr(cfg, "seat_timeout_seconds", 600) or 600),
bool(getattr(cfg, "schema_repair_enabled", True)),
)
except Exception: # noqa: BLE001
return 600.0, True
def _build_workflow(self, state: RoundtableState) -> Any:
seat_executors = [
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
for agent_id in state.seat_agent_ids
]
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
finalizer = FinalizerExecutor(self._adapter, state)
waiting_terminal = WaitingTerminalExecutor()
if state.mode == "group_chat":
coordinator = CoordinatorExecutor(self._adapter, state)
return create_workflow(
mode=state.mode,
coordinator=coordinator,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=state.seat_agent_ids,
)
planner = PlannerExecutor(self._adapter, state)
cross_review = CrossReviewExecutor(self._adapter, state)
return create_workflow(
mode="structured",
planner=planner,
cross_review=cross_review,
seat_executors=seat_executors,
approval_gate=approval_gate,
finalizer=finalizer,
waiting_terminal=waiting_terminal,
seat_agent_ids=state.seat_agent_ids,
)
async def _persist_checkpoint(self, state: RoundtableState, *, reason: str = "") -> None:
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
await self._adapter._emit_critical(
state.run_id,
make_workflow_checkpointed(
None,
state.current_round,
checkpoint_id=f"nodes-{len(state.completed_nodes)}",
reason=reason or f"stage={state.stage}",
),
)
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
"""Execute the workflow graph via the MAF SDK's streaming API.
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
events. Persists the RoundtableState checkpoint after each superstep.
"""
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
event_type = getattr(event, "type", "")
executor_id = getattr(event, "executor_id", None)
if event_type == "superstep_completed":
# Persist checkpoint after each superstep.
iteration = getattr(event, "iteration", None)
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
executor_id, state.current_round,
checkpoint_id=f"superstep-{iteration}",
reason="superstep_completed",
))
await self._persist_checkpoint(
state, reason=f"superstep_completed:{iteration}"
)
elif event_type in ("executor_completed", "executor_invoked"):
# Persist after each node so completed_nodes survive restarts.
if state.completed_nodes:
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
elif event_type == "executor_failed":
details = getattr(event, "details", None)
error_msg = str(details) if details else "executor failed"
# Soft seat failures are handled inside SeatExecutor; only hard
# failures (e.g. WriteGuardConfigurationError) abort the run.
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
# Always persist latest completed_nodes at end of stream.
await self._service.store.runs.update_run(state.run_id, {
"maf_checkpoint": state.to_dict(),
})
# Check if the workflow ended in a waiting_approval state.
if state.status == "waiting_approval" or state.pending_approvals:
state.status = "waiting_approval"
@@ -286,10 +341,22 @@ class WorkflowRunner:
for pa in state.pending_approvals:
approval_id = pa.get("approval_id", "")
action_id = pa.get("action_id", "")
# Extracted up-front so EVERY branch (rejected / failed / approved)
# can write the ledger transition with the correct agent_id.
agent_id = pa.get("agent_id", "")
thread_id = state.thread_map.get(agent_id, "")
tool_name = pa.get("tool_name", "")
approval = await self._service.store.approvals.get_approval(approval_id)
if approval is None:
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": "approval not found"})
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="failed",
agent_id=agent_id,
)
continue
approval_status = approval.get("status", "pending")
@@ -307,17 +374,32 @@ class WorkflowRunner:
decided_by=approval.get("decided_by", ""),
))
action_outcomes.append({"action_id": action_id, "status": "rejected"})
# Phase-3 P1-3: move the ledger row out of pending into the
# failed/unknown partition as rejected.
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="rejected",
agent_id=agent_id,
)
continue
if approval_status != "approved":
action_outcomes.append({"action_id": action_id, "status": "failed",
"error": f"unexpected approval status: {approval_status}"})
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=pa.get("tool_name", ""),
status="failed",
agent_id=agent_id,
)
continue
# Approved — execute deterministically via ToolExecutor.
agent_id = pa.get("agent_id", "")
thread_id = state.thread_map.get(agent_id, "")
tool_name = pa.get("tool_name", "")
# (agent_id / thread_id / tool_name are already extracted at the
# top of this loop iteration so every branch can use them.)
# Conditional transition (waiting_approval → approved). The
# authoritative at-most-once gate is the ToolExecutor's
@@ -378,11 +460,20 @@ class WorkflowRunner:
"error": outcome.get("error"),
})
state.fact_ledger["pending_actions"].append({
"action_id": action_id,
"tool_name": tool_name,
"outcome": outcome.get("status"),
})
out_status = outcome.get("status")
ledger_status = {
"succeeded": "executed",
"failed": "failed",
"unknown": "unknown",
"skipped": "executed",
}.get(str(out_status), "unknown")
record_action_transition(
state.fact_ledger,
action_id=action_id,
tool_name=tool_name,
status=ledger_status,
agent_id=agent_id,
)
# P0-3: determine parent run status from action outcomes.
statuses = {o.get("status") for o in action_outcomes}
@@ -451,6 +542,76 @@ class WorkflowRunner:
return await self._service.store.runs.get_run(run_id)
async def recover(self, run_id: str) -> dict[str, Any] | None:
"""Graph-level recovery after process restart (MAF-03-07).
Reloads ``RoundtableState`` from ``maf_checkpoint`` and re-enters the
structured workflow. Executors whose ids are in ``completed_nodes``
short-circuit without calling DeerFlow again.
"""
t0 = time.perf_counter()
try:
import agent_framework # noqa: F401
except ImportError as exc:
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
run = await self._service.store.runs.get_run(run_id)
if run is None:
return None
if run["status"] not in ("running", "waiting_approval"):
logger.info("recover(%s): status=%s — nothing to recover", run_id, run["status"])
return run
checkpoint = run.get("maf_checkpoint")
if not checkpoint:
logger.warning("recover(%s): no checkpoint", run_id)
return run
state = RoundtableState.from_dict(checkpoint)
session = await self._service.store.sessions.get_session(state.session_id, user_id=None)
if session is None:
return run
# Ensure threads still bound.
participants = await self._service.store.participants.list_participants(state.session_id)
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
state.thread_map.update(thread_map)
if run["status"] == "waiting_approval":
# Approval path stays on the existing resume() — recover only
# continues orchestration for running graphs.
return run
await self._service.store.runs.update_run(run_id, {"status": "running"})
try:
workflow = self._build_workflow(state)
await self._run_workflow(workflow, state)
latency = int((time.perf_counter() - t0) * 1000)
final_status = state.status
update_fields: dict[str, Any] = {
"status": final_status,
"latency_ms": latency,
"maf_checkpoint": state.to_dict(),
"token_usage": _aggregate_token_usage(state.turn_results),
}
if final_status in ("completed", "failed", "cancelled"):
update_fields["finished_at"] = datetime.now(UTC)
await self._service.store.runs.update_run(run_id, update_fields)
session_fields: dict[str, Any] = {"status": final_status}
if final_status == "completed":
session_fields["final_result"] = state.final_result
if final_status != "waiting_approval":
session_fields["fact_ledger"] = state.fact_ledger
await self._service.store.sessions.update_session(
state.session_id, user_id=self._user_id, fields=session_fields
)
except Exception as exc: # noqa: BLE001
await self._fail_run(run_id, state.session_id, t0, type(exc).__name__, str(exc))
return await self._service.store.runs.get_run(run_id)
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
"""Aggregate token usage across all seat turns into a run-level summary."""
@@ -563,6 +563,8 @@ maf_roundtable:
default_write_policy: confirm_high_risk # auto | confirm_high_risk | confirm_all_writes
default_max_rounds: 8 # 群聊模式轮次硬上限
admin_diagnostic_access: true # 管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)
seat_timeout_seconds: 600 # 结构化流程单席位超时(秒)
schema_repair_enabled: true # 席位 JSON 校验失败时允许一次定向修复
# ── WriteGuard 中间件配置 ────────────────────────────────────────────────────
# 与 maf_roundtable 联动:MAF 会商运行时,WriteGuard 中间件拦截外部写工具调用,
@@ -62,6 +62,18 @@ class MafRoundtableConfig(BaseModel):
default=True,
description="管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)",
)
# Phase 3: per-seat wall-clock timeout (seconds) for structured expert turns.
seat_timeout_seconds: int = Field(
default=600,
ge=30,
le=7200,
description="结构化流程中单个专家席位的超时秒数;超时记为 partial_failure 并继续其余席位",
)
# Phase 3: allow one directed format-repair turn when seat JSON fails schema.
schema_repair_enabled: bool = Field(
default=True,
description="席位输出未通过 SeatTurnOutput 校验时是否允许一次定向格式修复",
)
__all__ = [
@@ -30,6 +30,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import (
CheckConstraint,
ForeignKey,
Integer,
String,
@@ -306,13 +307,34 @@ class MafRoundtableSeatRunRow(Base):
the system can still locate and cancel outstanding DeerFlow runs, and the
MAF checkpoint can reference which seat turns completed.
Unique constraint: ``(maf_run_id, agent_id, turn_index)`` — one DeerFlow run
per seat per turn.
Phase-3 unique constraint: ``(maf_run_id, agent_id, turn_index,
execution_stage)``. ``turn_index`` is the business round (kept stable for
frontend display / approval matching / event tracing); ``execution_stage``
disambiguates the distinct DeerFlow executions that happen within one
business round (planner / expert / format_repair / cross_review /
finalizer / group_chat). The business turn_index is NEVER mutated to dodge
a collision.
"""
__tablename__ = "maf_roundtable_seat_runs"
__table_args__ = (
UniqueConstraint("maf_run_id", "agent_id", "turn_index", name="uq_maf_seat_runs_run_agent_turn"),
UniqueConstraint(
"maf_run_id",
"agent_id",
"turn_index",
"execution_stage",
name="uq_maf_seat_runs_run_agent_turn_stage",
),
# Phase-3 P1-2: enforce the closed execution_stage enum at the DB layer.
# Mirrors seat_runs.VALID_EXECUTION_STAGES — a typo (e.g. "exprt") would
# otherwise silently bypass the four-column idempotency key by creating
# a brand-new stage. This CHECK keeps both fresh creates and any raw
# migration / manual SQL honest. Keep in sync with VALID_EXECUTION_STAGES.
CheckConstraint(
"execution_stage IN ('legacy','planner','expert','format_repair',"
"'cross_review','cross_review_repair','finalizer','group_chat')",
name="ck_maf_seat_runs_execution_stage",
),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
@@ -321,12 +343,24 @@ class MafRoundtableSeatRunRow(Base):
)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
turn_index: Mapped[int] = mapped_column(Integer, nullable=False)
# Phase-3: which kind of DeerFlow execution this row represents. Pre-Phase-3
# rows are migrated to ``legacy``. Closed enum (see seat_runs.VALID_EXECUTION_STAGES):
# planner | expert | format_repair | cross_review | cross_review_repair |
# finalizer | group_chat | legacy.
execution_stage: Mapped[str] = mapped_column(
String(32), nullable=False, default="legacy", server_default="legacy"
)
deerflow_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
deerflow_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
sandbox_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
workflow_checkpoint_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# pending | running | completed | failed | cancelled
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
# Phase-3 P0-1: a sanitized, replayable snapshot of the SeatResponse written
# atomically with the completed transition. Survives a crash that happens
# after the seat run is marked completed but before the workflow checkpoint
# is persisted, so recovery never needs to re-call DeerFlow.
result_snapshot: Mapped[dict | None] = mapped_column(PortableJSON, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
@@ -12,6 +12,7 @@ from typing import Any
from uuid import uuid4
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.maf_roundtable.model import MafRoundtableSeatRunRow
@@ -21,17 +22,68 @@ def _now() -> datetime:
return datetime.now(UTC)
# ── execution_stage enum (Phase-3 P1-2) ─────────────────────────────────────
# The fixed set of distinct DeerFlow executions that may occur within one
# business round (turn_index). The four-column idempotency key
# (maf_run_id, agent_id, turn_index, execution_stage) only works if stages
# are a closed set — a typo would silently create a new stage and bypass the
# dedup. Validate on write so a misspelling raises loudly instead.
LEGACY = "legacy"
PLANNER = "planner"
EXPERT = "expert"
FORMAT_REPAIR = "format_repair"
CROSS_REVIEW = "cross_review"
CROSS_REVIEW_REPAIR = "cross_review_repair"
FINALIZER = "finalizer"
GROUP_CHAT = "group_chat"
VALID_EXECUTION_STAGES: frozenset[str] = frozenset({
LEGACY,
PLANNER,
EXPERT,
FORMAT_REPAIR,
CROSS_REVIEW,
CROSS_REVIEW_REPAIR,
FINALIZER,
GROUP_CHAT,
})
class InvalidExecutionStageError(ValueError):
"""Raised when an execution_stage outside the closed enum is written."""
class SeatRunNotFoundError(LookupError):
"""Raised when a seat run expected by the four-tuple does not exist."""
class SeatRunWaitTimeout(TimeoutError):
"""Raised when waiting for a seat run to reach a terminal state times out."""
def _validate_stage(execution_stage: str) -> str:
if execution_stage not in VALID_EXECUTION_STAGES:
raise InvalidExecutionStageError(
f"execution_stage={execution_stage!r} is not one of the allowed "
f"stages {sorted(VALID_EXECUTION_STAGES)}; a typo here would bypass "
f"the four-column idempotency key."
)
return execution_stage
def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
return {
"id": row.id,
"maf_run_id": row.maf_run_id,
"agent_id": row.agent_id,
"turn_index": row.turn_index,
"execution_stage": row.execution_stage,
"deerflow_run_id": row.deerflow_run_id,
"deerflow_thread_id": row.deerflow_thread_id,
"sandbox_id": row.sandbox_id,
"workflow_checkpoint_id": row.workflow_checkpoint_id,
"status": row.status,
"result_snapshot": row.result_snapshot,
"started_at": row.started_at.isoformat() if isinstance(row.started_at, datetime) else row.started_at,
"finished_at": row.finished_at.isoformat() if isinstance(row.finished_at, datetime) else row.finished_at,
"error": row.error,
@@ -40,6 +92,12 @@ def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
}
# Seat-run lifecycle statuses that mean "still in flight" — a second attempt to
# start the same DeerFlow execution must NOT launch another run while one of
# these exists.
_ACTIVE_STATUSES = ("pending", "running")
class MafSeatRunRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -50,14 +108,17 @@ class MafSeatRunRepository:
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str = "legacy",
deerflow_thread_id: str | None = None,
) -> dict[str, Any]:
_validate_stage(execution_stage)
now = _now()
row = MafRoundtableSeatRunRow(
id=uuid4().hex,
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
deerflow_thread_id=deerflow_thread_id,
status="pending",
created_at=now,
@@ -68,6 +129,171 @@ class MafSeatRunRepository:
await session.commit()
return _to_dict(row)
async def get_by_four_tuple(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
) -> dict[str, Any] | None:
"""Look up a seat run by its Phase-3 identity key (no DeerFlow side effects)."""
stmt = select(MafRoundtableSeatRunRow).where(
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
MafRoundtableSeatRunRow.agent_id == agent_id,
MafRoundtableSeatRunRow.turn_index == turn_index,
MafRoundtableSeatRunRow.execution_stage == execution_stage,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def wait_for_terminal_seat_run(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
timeout_seconds: float,
poll_interval_seconds: float = 0.1,
) -> dict[str, Any]:
"""Block until the seat run identified by the four-tuple reaches a terminal status.
Terminal statuses: completed, failed, cancelled, cancel_failed, cancel_unknown.
Non-terminal (still in-flight): pending, running, cancelling.
Raises:
SeatRunNotFoundError: if no row exists for the four-tuple.
SeatRunWaitTimeout: if the deadline is reached while still non-terminal.
"""
import asyncio
import time
_TERMINAL = ("completed", "failed", "cancelled", "cancel_failed", "cancel_unknown")
deadline = time.monotonic() + timeout_seconds
while True:
row = await self.get_by_four_tuple(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if row is None:
raise SeatRunNotFoundError(
f"seat run not found: maf_run_id={maf_run_id!r} agent_id={agent_id!r} "
f"turn_index={turn_index} execution_stage={execution_stage!r}"
)
if row["status"] in _TERMINAL:
return row
if time.monotonic() >= deadline:
raise SeatRunWaitTimeout(
f"timed out waiting for seat run {row['id']!r} "
f"(stage={execution_stage!r}, status={row['status']!r}) "
f"after {timeout_seconds}s"
)
await asyncio.sleep(poll_interval_seconds)
async def get_or_create_seat_run(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
deerflow_thread_id: str | None = None,
) -> tuple[dict[str, Any], bool]:
"""Atomic idempotent seat-run acquisition (Phase-3 P0-1).
Race-free: tries to INSERT first; on a four-column unique-key conflict
it rolls back the short transaction and reads the existing row. This is
strictly stronger than "query-then-insert" because two concurrent
callers that both miss the lookup would otherwise both insert — only
the DB constraint decides the winner, and the loser reads the winner's
row.
Because the winner's transaction may not have committed by the time the
loser's post-conflict read runs (session isolation), the loser retries
the read a bounded number of times with a short backoff before giving
up. ``IntegrityError`` is NEVER propagated to the caller on a four-tuple
conflict.
Returns ``(seat_run, created)``:
- ``created=True`` → caller OWNS this execution and may start DeerFlow.
- ``created=False`` → a row already exists; the caller must inspect its
``status`` and must NOT start a second DeerFlow run for it.
"""
_validate_stage(execution_stage)
now = _now()
row = MafRoundtableSeatRunRow(
id=uuid4().hex,
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
deerflow_thread_id=deerflow_thread_id,
status="pending",
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError:
# Lost the race (or replayed an existing four-tuple). Roll back
# this short transaction, then read the winner's row. The winner
# may not have committed yet (session isolation), so retry a few
# times with a short backoff before treating this as a hard
# error.
await session.rollback()
existing = await self._await_existing(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if existing is None:
# The conflict was NOT on our four-tuple (e.g. a FK
# violation) — surface it as a clean error rather than
# swallowing it.
raise
return existing, False
return _to_dict(row), True
async def _await_existing(
self,
*,
maf_run_id: str,
agent_id: str,
turn_index: int,
execution_stage: str,
retries: int = 10,
backoff_s: float = 0.01,
) -> dict[str, Any] | None:
"""Read the winner's row, retrying briefly until it is visible.
Under true concurrency the winner's COMMIT may land after the loser's
post-rollback SELECT; the loser cannot see an uncommitted row, so we
poll with a tiny backoff until the winner's transaction commits (or the
retry budget is exhausted, in which case we return None so the caller
can decide it was a non-four-tuple conflict).
"""
import asyncio
for _ in range(max(1, retries)):
existing = await self.get_by_four_tuple(
maf_run_id=maf_run_id,
agent_id=agent_id,
turn_index=turn_index,
execution_stage=execution_stage,
)
if existing is not None:
return existing
await asyncio.sleep(backoff_s)
return None
async def mark_running(
self, seat_run_id: str, *, deerflow_run_id: str, sandbox_id: str | None = None
) -> None:
@@ -85,17 +311,29 @@ class MafSeatRunRepository:
)
await session.commit()
async def mark_completed(self, seat_run_id: str, *, checkpoint_id: str | None = None) -> None:
async def mark_completed(
self,
seat_run_id: str,
*,
checkpoint_id: str | None = None,
result_snapshot: dict[str, Any] | None = None,
) -> None:
"""Mark a seat run completed, optionally persisting a sanitized result
snapshot so it can be replayed after a crash even if the workflow
checkpoint was not yet written."""
values: dict[str, Any] = {
"status": "completed",
"workflow_checkpoint_id": checkpoint_id,
"finished_at": _now(),
"updated_at": _now(),
}
if result_snapshot is not None:
values["result_snapshot"] = result_snapshot
async with self._sf() as session:
await session.execute(
update(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.id == seat_run_id)
.values(
status="completed",
workflow_checkpoint_id=checkpoint_id,
finished_at=_now(),
updated_at=_now(),
)
.values(**values)
)
await session.commit()
@@ -163,7 +401,10 @@ class MafSeatRunRepository:
stmt = (
select(MafRoundtableSeatRunRow)
.where(MafRoundtableSeatRunRow.maf_run_id == maf_run_id)
.order_by(MafRoundtableSeatRunRow.turn_index.asc())
.order_by(
MafRoundtableSeatRunRow.turn_index.asc(),
MafRoundtableSeatRunRow.execution_stage.asc(),
)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
@@ -0,0 +1,402 @@
"""Phase 3 migration for the MAF roundtable experiment.
Idempotent migration that brings a Phase-2 deployment up to Phase-3 schema:
1. Add the non-null ``execution_stage`` column to ``maf_roundtable_seat_runs``
(existing rows default to ``legacy``).
2. Replace the Phase-2 three-column unique constraint
``(maf_run_id, agent_id, turn_index)`` with the Phase-3 four-column
constraint ``(maf_run_id, agent_id, turn_index, execution_stage)`` so the
same business round can hold multiple distinct DeerFlow executions
(planner / expert / format_repair / cross_review / finalizer / group_chat)
while each stage is still unique per (run, agent, turn).
The business ``turn_index`` is NEVER mutated to dodge a collision — it stays
the stable business round used by the frontend, approval matching and event
tracing. ``execution_stage`` is the new disambiguator.
Dialect handling
----------------
- **PostgreSQL / MySQL**: ``ALTER TABLE`` to add the column, then ``DROP`` the
old unique constraint/index and ``ADD`` the new one. Each step is guarded by
introspection so re-runs are no-ops.
- **SQLite**: a unique constraint cannot be dropped in place, so the table is
rebuilt: a temp table with the new shape is created, data is copied (existing
rows get ``execution_stage='legacy'``), the old table is dropped, and the
temp is renamed. This is the only safe way to change a SQLite unique
constraint; it is fully idempotent because the rebuild only runs when the
``execution_stage`` column is absent or the old constraint is still present.
**Deployment order**: run AFTER the Phase-2 migration and BEFORE enabling the
Phase-3 code path (the four-column constraint is required for the new
``execution_stage`` writes to coexist within one turn).
Usage (from ``offline-backend-20260512/backend/``)::
# Preview (no writes)
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase3.py --dry-run
# Execute
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase3.py
"""
from __future__ import annotations
import argparse
import asyncio
import logging
import sys
from typing import Any
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger("migrate_maf_roundtable_phase3")
OLD_CONSTRAINT_NAME = "uq_maf_seat_runs_run_agent_turn"
NEW_CONSTRAINT_NAME = "uq_maf_seat_runs_run_agent_turn_stage"
TABLE = "maf_roundtable_seat_runs"
async def _migrate(dry_run: bool) -> int:
from deerflow.config import get_app_config
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
config = get_app_config()
await init_engine_from_config(config.database)
sf = get_session_factory()
if sf is None:
logger.error("Database backend is 'memory' — nothing to migrate.")
return 0
engine = sf.kw["bind"] # the async engine
async with engine.begin() as conn:
await conn.run_sync(_do_migrate_sync, dry_run=dry_run, logger=logger, changes_ref=[0])
await close_engine()
logger.info("Migration complete (dry_run=%s).", dry_run)
return 0
def _do_migrate_sync(
sync_conn: Any,
*,
dry_run: bool,
logger: logging.Logger,
changes_ref: list[int],
) -> None:
"""Run all migration steps on a sync connection (called via run_sync)."""
from sqlalchemy import inspect, text
insp = inspect(sync_conn)
dialect_name = sync_conn.dialect.name
def _exec(sql: str) -> None:
if dry_run:
logger.info(" [dry-run] would execute: %s", sql.strip()[:300])
return
sync_conn.execute(text(sql))
changes_ref[0] += 1
if TABLE not in insp.get_table_names():
logger.info("Table %s does not exist — Phase-3 migration is a no-op (table will be created by the ORM with the new schema).", TABLE)
return
columns = {c["name"] for c in insp.get_columns(TABLE)}
has_stage_col = "execution_stage" in columns
has_snapshot_col = "result_snapshot" in columns
# ── 1a. Add execution_stage column (additive; old rows → 'legacy') ─────
if not has_stage_col:
logger.info("Adding column execution_stage to %s", TABLE)
_exec(_ddl_add_execution_stage(dialect_name))
else:
logger.info("Column execution_stage already exists on %s — skip.", TABLE)
# ── 2. Replace the unique constraint (3-tuple → 4-tuple) ───────────────
# NOTE: introspection is re-fetched after the column add so the inspector
# reflects the new column within this same transaction.
insp = inspect(sync_conn)
old_constraint = _find_unique_constraint(insp, TABLE, expected_cols={"maf_run_id", "agent_id", "turn_index"})
new_constraint = _find_unique_constraint(insp, TABLE, expected_cols={"maf_run_id", "agent_id", "turn_index", "execution_stage"})
has_new_unique = new_constraint is not None
has_stage_check = _has_check_constraint(insp, TABLE, "ck_maf_seat_runs_execution_stage")
if dialect_name == "sqlite":
needs_rebuild = (
not has_new_unique
or not has_snapshot_col
or not has_stage_check
)
if needs_rebuild:
logger.info("SQLite: rebuilding %s (has_new_unique=%s, has_snapshot=%s, has_check=%s).",
TABLE, has_new_unique, has_snapshot_col, has_stage_check)
for stmt in _sqlite_rebuild_seat_runs(
source_has_stage=has_stage_col,
source_has_snapshot=has_snapshot_col,
):
_exec(stmt)
else:
logger.info("SQLite: %s already fully migrated — no-op.", TABLE)
logger.info("Migration steps: %d applied.", changes_ref[0])
return
# PostgreSQL / MySQL
if not has_new_unique:
if old_constraint is not None:
old_name = old_constraint.get("name") or OLD_CONSTRAINT_NAME
old_kind = old_constraint.get("kind")
if old_kind == "index":
logger.info("Dropping old unique INDEX %s (kind=index)", old_name)
_exec(_ddl_drop_index(old_name))
else:
logger.info("Dropping old unique CONSTRAINT %s (kind=constraint)", old_name)
_exec(_ddl_drop_constraint(dialect_name, old_name))
else:
logger.info(
"Old unique object not found by introspection — migration assumes "
"the four-column constraint is being created fresh."
)
logger.info("Adding new four-column unique constraint %s", NEW_CONSTRAINT_NAME)
_exec(_ddl_add_new_constraint(dialect_name))
if not has_snapshot_col:
insp = inspect(sync_conn)
cols_now = {c["name"] for c in insp.get_columns(TABLE)}
if "result_snapshot" not in cols_now:
logger.info("Adding column result_snapshot to %s", TABLE)
_exec(_ddl_add_result_snapshot(dialect_name))
insp = inspect(sync_conn)
if not _has_check_constraint(insp, TABLE, "ck_maf_seat_runs_execution_stage"):
logger.info("Adding CHECK constraint ck_maf_seat_runs_execution_stage")
_exec(_ddl_add_execution_stage_check(dialect_name))
logger.info("Migration steps: %d applied.", changes_ref[0])
def _find_unique_constraint(insp: Any, table: str, *, expected_cols: set[str]) -> dict[str, Any] | None:
"""Return the unique constraint (or unique index) matching ``expected_cols``.
Looks at both table-level unique constraints AND unique indexes — Phase-2
created the constraint via ``CREATE UNIQUE INDEX`` which SQLAlchemy reports
as an index, while the ORM model declares it as a UniqueConstraint (reported
as a constraint). We accept either form so the migration is robust to how
the table was originally created.
"""
for uc in insp.get_unique_constraints(table):
if set(uc.get("column_names") or []) == expected_cols:
return {"name": uc.get("name"), "kind": "constraint"}
for idx in insp.get_indexes(table):
if idx.get("unique") and set(idx.get("column_names") or []) == expected_cols:
return {"name": idx.get("name"), "kind": "index"}
return None
def _has_check_constraint(insp: Any, table: str, name: str) -> bool:
"""True if a CHECK constraint with the given name exists on ``table``.
SQLAlchemy's inspector exposes CHECK constraints via ``get_check_constraints``
on PostgreSQL / MySQL / modern SQLite. We match by name; if the dialect
cannot list checks we conservatively return False so the (idempotent) ADD
runs once — a duplicate-name error there is harmless and caught by the
``IF NOT EXISTS``-style guard in the caller's re-introspection.
"""
getter = getattr(insp, "get_check_constraints", None)
if getter is None:
return False
try:
for ck in getter(table):
if ck.get("name") == name:
return True
except Exception: # noqa: BLE001
return False
return False
# ── DDL templates (portable) ────────────────────────────────────────────────
def _ddl_add_execution_stage(dialect: str) -> str:
"""Add the execution_stage column, back-filling existing rows to 'legacy'.
Adding NOT NULL with a server default is safe on all three dialects: the
default populates existing rows during the ALTER, then stays for new ones.
"""
if dialect == "mysql":
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
)
if dialect == "postgresql":
# Postgres needs the default, and we set it server-side so existing
# rows back-fill automatically.
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
)
# SQLite < 3.35 does not support adding a NOT NULL column with a default
# via ALTER TABLE in the same way; but modern SQLite (3.35+, 2021+) does.
# The default 'legacy' back-fills existing rows.
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
)
def _ddl_add_result_snapshot(dialect: str) -> str:
"""Add the nullable ``result_snapshot`` JSON column (Phase-3 P0-1b).
Stores a sanitized, replayable snapshot of the SeatResponse written
atomically with the completed transition. Nullable so the ALTER is safe on
all dialects (existing rows get NULL).
"""
if dialect == "mysql":
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN result_snapshot JSON NULL"
)
if dialect == "postgresql":
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN result_snapshot JSON NULL"
)
# SQLite — reached only when not rebuilding; use TEXT for JSON.
return (
"ALTER TABLE maf_roundtable_seat_runs "
"ADD COLUMN result_snapshot TEXT"
)
def _ddl_drop_constraint(dialect: str, name: str) -> str:
if dialect == "mysql":
# MySQL uses DROP INDEX for both unique constraints and unique indexes.
return f"ALTER TABLE maf_roundtable_seat_runs DROP INDEX {name}"
# PostgreSQL / ANSI
return f"ALTER TABLE maf_roundtable_seat_runs DROP CONSTRAINT {name}"
def _ddl_drop_index(name: str) -> str:
return f"DROP INDEX {name}"
def _ddl_add_new_constraint(dialect: str) -> str:
new = NEW_CONSTRAINT_NAME
cols = "maf_run_id, agent_id, turn_index, execution_stage"
if dialect == "mysql":
return (
f"ALTER TABLE maf_roundtable_seat_runs "
f"ADD UNIQUE INDEX {new} ({cols})"
)
# PostgreSQL — table constraint
return (
f"ALTER TABLE maf_roundtable_seat_runs "
f"ADD CONSTRAINT {new} UNIQUE ({cols})"
)
def _ddl_add_execution_stage_check(dialect: str) -> str:
"""Add the closed-enum CHECK constraint on execution_stage.
Mirrors ``seat_runs.VALID_EXECUTION_STAGES`` so the DB itself rejects a typo
(e.g. 'exprt') that would otherwise create a brand-new stage and bypass the
four-column idempotency key. Keep the value list in sync with
``VALID_EXECUTION_STAGES``.
"""
allowed = (
"'legacy','planner','expert','format_repair',"
"'cross_review','cross_review_repair','finalizer','group_chat'"
)
if dialect == "mysql":
# MySQL 8 accepts inline CHECK; older MySQL silently ignores it, but the
# app-level validation in seat_runs._validate_stage is the real guard.
return (
f"ALTER TABLE maf_roundtable_seat_runs "
f"ADD CONSTRAINT ck_maf_seat_runs_execution_stage "
f"CHECK (execution_stage IN ({allowed}))"
)
# PostgreSQL / ANSI
return (
f"ALTER TABLE maf_roundtable_seat_runs "
f"ADD CONSTRAINT ck_maf_seat_runs_execution_stage "
f"CHECK (execution_stage IN ({allowed}))"
)
def _sqlite_rebuild_seat_runs(
*,
source_has_stage: bool = False,
source_has_snapshot: bool = False,
) -> list[str]:
"""Rebuild the table in SQLite to swap the unique constraint.
SQLite's ``ALTER TABLE`` cannot drop a constraint, so the portable recipe is:
create a new table with the desired shape, copy data over, drop the old
table, and rename. Indexes are recreated on the new table.
``source_has_stage`` / ``source_has_snapshot`` control the INSERT SELECT
expressions so an intermediate table (already has stage and/or snapshot)
does NOT lose data by being overwritten with defaults.
"""
allowed = (
"'legacy','planner','expert','format_repair',"
"'cross_review','cross_review_repair','finalizer','group_chat'"
)
source_stage = "execution_stage" if source_has_stage else "'legacy'"
source_snapshot = "result_snapshot" if source_has_snapshot else "NULL"
return [
f"""
CREATE TABLE maf_roundtable_seat_runs__phase3 (
id VARCHAR(64) NOT NULL PRIMARY KEY,
maf_run_id VARCHAR(64) NOT NULL,
agent_id VARCHAR(64) NOT NULL,
turn_index INTEGER NOT NULL,
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
deerflow_run_id VARCHAR(64),
deerflow_thread_id VARCHAR(64),
sandbox_id VARCHAR(64),
workflow_checkpoint_id VARCHAR(128),
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
result_snapshot TEXT,
started_at DATETIME,
finished_at DATETIME,
error TEXT,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
CONSTRAINT {NEW_CONSTRAINT_NAME} UNIQUE (maf_run_id, agent_id, turn_index, execution_stage),
CONSTRAINT ck_maf_seat_runs_execution_stage CHECK (execution_stage IN ({allowed}))
)
""",
f"""
INSERT INTO maf_roundtable_seat_runs__phase3 (
id, maf_run_id, agent_id, turn_index, execution_stage,
deerflow_run_id, deerflow_thread_id, sandbox_id, workflow_checkpoint_id,
status, result_snapshot, started_at, finished_at, error, created_at, updated_at
)
SELECT
id, maf_run_id, agent_id, turn_index, {source_stage},
deerflow_run_id, deerflow_thread_id, sandbox_id, workflow_checkpoint_id,
status, {source_snapshot}, started_at, finished_at, error, created_at, updated_at
FROM maf_roundtable_seat_runs
""",
"DROP TABLE maf_roundtable_seat_runs",
"ALTER TABLE maf_roundtable_seat_runs__phase3 RENAME TO maf_roundtable_seat_runs",
"CREATE INDEX ix_maf_seat_runs_maf_run_id ON maf_roundtable_seat_runs (maf_run_id)",
"CREATE INDEX ix_maf_seat_runs_agent_id ON maf_roundtable_seat_runs (agent_id)",
"CREATE INDEX ix_maf_seat_runs_deerflow_run_id ON maf_roundtable_seat_runs (deerflow_run_id)",
"CREATE INDEX ix_maf_seat_runs_status ON maf_roundtable_seat_runs (status)",
]
def main() -> int:
parser = argparse.ArgumentParser(description="Phase 3 migration for MAF roundtable seat runs")
parser.add_argument("--dry-run", action="store_true", help="Preview only; do not write to the database")
args = parser.parse_args()
return asyncio.run(_migrate(args.dry_run))
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,108 @@
"""Unit tests for the Phase-3 shared fact ledger."""
from __future__ import annotations
import pytest
from app.maf_roundtable.fact_ledger import (
assert_ledger_invariants,
empty_ledger,
executed_action_ids,
merge_seat_output,
normalize_ledger,
proposed_action_ids,
record_action_transition,
record_tool_result,
)
def test_empty_ledger_has_all_partitions():
ledger = empty_ledger()
for key in (
"confirmed_facts",
"evidence",
"tool_results",
"executed_actions",
"pending_actions",
"failed_or_unknown_actions",
"conflicts",
"proposed_actions",
):
assert key in ledger
assert ledger[key] == []
def test_normalize_legacy_checkpoint():
legacy = {
"confirmed_facts": ["a"],
"tool_results": [{"tool_name": "t", "status": "completed"}],
"conflicts": [],
"pending_actions": [],
}
n = normalize_ledger(legacy)
assert n["confirmed_facts"] == ["a"]
assert n["executed_actions"] == []
assert n["evidence"] == []
def test_merge_seat_output_keeps_proposed_out_of_executed():
ledger = empty_ledger()
merge_seat_output(
ledger,
{
"summary": "s",
"findings": ["fact-1"],
"evidence_refs": ["doc:1"],
"proposed_actions": [{"text": "create ticket", "action_id": "prop-1"}],
"executed_actions": [{"text": "forged", "action_id": "forge-1"}],
"conflicts": ["c1"],
},
agent_id="seat-a",
)
# Phase-3: seat findings land in candidate_facts (unverified claims), NOT
# confirmed_facts. Promotion to confirmed happens only after cross-review.
assert ledger["confirmed_facts"] == []
assert any(
c.get("text") == "fact-1" for c in ledger["candidate_facts"] if isinstance(c, dict)
)
assert "prop-1" in proposed_action_ids(ledger)
assert "forge-1" not in executed_action_ids(ledger)
assert_ledger_invariants(ledger)
def test_record_action_transition_pending_to_executed():
ledger = empty_ledger()
record_action_transition(
ledger, action_id="a1", tool_name="create_ticket", status="pending", agent_id="s1"
)
assert len(ledger["pending_actions"]) == 1
record_action_transition(
ledger, action_id="a1", tool_name="create_ticket", status="executed", agent_id="s1"
)
assert ledger["pending_actions"] == []
assert executed_action_ids(ledger) == {"a1"}
assert_ledger_invariants(ledger)
def test_record_action_transition_failed_not_executed():
ledger = empty_ledger()
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="pending")
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="unknown")
assert executed_action_ids(ledger) == set()
assert any(e.get("action_id") == "a2" for e in ledger["failed_or_unknown_actions"])
def test_record_tool_result_status_clamp():
ledger = empty_ledger()
record_tool_result(ledger, tool_name="web_search", status="completed", agent_id="a")
record_tool_result(ledger, tool_name="x", status="bogus", agent_id="a")
assert ledger["tool_results"][0]["status"] == "completed"
assert ledger["tool_results"][1]["status"] == "unknown"
assert_ledger_invariants(ledger)
def test_assert_rejects_bad_tool_status():
ledger = empty_ledger()
ledger["tool_results"].append({"tool_name": "t", "status": "weird"})
with pytest.raises(AssertionError):
assert_ledger_invariants(ledger)
@@ -0,0 +1,466 @@
"""Phase-3 acceptance tests for the fact ledger + cross-review wiring.
Covers P1-2 (candidate_facts separation), P1-3 (approval ledger integrity),
and P1-1 (CrossReviewOutput validation + deterministic degradation).
These are pure-unit tests over the fact_ledger / seat_output / workflow_executors
helpers — no DB, no real DeerFlow.
"""
from __future__ import annotations
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from app.maf_roundtable.fact_ledger import (
candidate_id_map,
confirm_candidate_facts,
empty_ledger,
merge_seat_output,
record_action_transition,
)
from app.maf_roundtable.seat_output import parse_cross_review_output
# ── P1-2: seat findings → candidate_facts, NOT confirmed_facts ─────────────
def test_seat_findings_land_in_candidate_facts_not_confirmed():
"""A seat's `findings` must NEVER be auto-promoted to confirmed_facts —
weak model output is a claim, not a fact."""
ledger = empty_ledger()
merge_seat_output(ledger, {"findings": ["市场增长10%", "竞品A领先"]}, agent_id="seat-1")
assert len(ledger["candidate_facts"]) == 2
assert ledger["confirmed_facts"] == [], "seat findings must not be confirmed facts"
# Each candidate gets a server-generated candidate_id.
for c in ledger["candidate_facts"]:
assert "candidate_id" in c and c["candidate_id"]
assert c["confirmed"] is False
assert c["agent_id"] == "seat-1"
assert c["source"] == "seat_finding"
def test_candidate_facts_promoted_only_after_cross_review_confirmation():
"""Promotion to confirmed_facts happens only via confirm_candidate_facts,
and only for ids that actually exist in candidate_facts AND cite evidence
the ledger has backing for."""
from app.maf_roundtable.fact_ledger import record_tool_result
ledger = empty_ledger()
merge_seat_output(ledger, {"findings": ["事实A", "事实B"]}, agent_id="seat-1")
ids = [c["candidate_id"] for c in ledger["candidate_facts"]]
assert len(ids) == 2
# Record real evidence so the confirmation can cite it (Phase-3 P1-1).
record_tool_result(
ledger, tool_name="web_search", status="completed",
agent_id="seat-1", turn_index=1,
)
# Model tries to confirm one real id + one HALLUCINATED id, citing the
# real evidence for both.
tool_ref = "tool:web_search:1"
confirm_candidate_facts(
ledger,
confirmed_candidate_ids=[ids[0], "nonexistent-id"],
confirmed_by="manager-1",
evidence_refs=[tool_ref],
)
assert len(ledger["confirmed_facts"]) == 1 # only the real one
assert ledger["confirmed_facts"][0]["candidate_id"] == ids[0]
assert ledger["confirmed_facts"][0]["confirmed_by"] == "manager-1"
assert ledger["confirmed_facts"][0]["source"] == "cross_review_confirmed"
# The confirmed candidate is marked confirmed=True in place.
assert candidate_id_map(ledger)[ids[0]]["confirmed"] is True
assert candidate_id_map(ledger)[ids[1]]["confirmed"] is False
def test_candidate_not_promoted_without_ledger_backed_evidence():
"""Phase-3 P1-1: confirmation requires evidence that resolves to a ledger
entry. A candidate cited with an invented evidence ref (not recorded in the
ledger) is REJECTED and stays a candidate."""
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
ledger = empty_ledger()
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
cid = ledger["candidate_facts"][0]["candidate_id"]
# No evidence recorded in the ledger; the model cites an invented ref.
confirm_candidate_facts(
ledger,
confirmed_candidate_ids=[cid],
confirmed_by="manager-1",
evidence_refs=["invented-evidence-not-in-ledger"],
)
assert ledger["confirmed_facts"] == [], "candidate promoted with non-ledger evidence"
assert candidate_id_map(ledger)[cid]["confirmed"] is False
def test_candidate_not_promoted_with_no_evidence_at_all():
"""Phase-3 P1-1: a confirmation that supplies no evidence_refs is rejected."""
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
ledger = empty_ledger()
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
cid = ledger["candidate_facts"][0]["candidate_id"]
confirm_candidate_facts(
ledger,
confirmed_candidate_ids=[cid],
confirmed_by="manager-1",
evidence_refs=None,
)
assert ledger["confirmed_facts"] == []
assert candidate_id_map(ledger)[cid]["confirmed"] is False
def test_confirm_is_idempotent():
"""Confirming the same candidate_id twice does not duplicate it."""
from app.maf_roundtable.fact_ledger import record_tool_result
ledger = empty_ledger()
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
cid = ledger["candidate_facts"][0]["candidate_id"]
record_tool_result(
ledger, tool_name="search", status="completed",
agent_id="seat-1", turn_index=1,
)
ref = "tool:search:1"
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
assert len(ledger["confirmed_facts"]) == 1
def test_empty_ledger_has_candidate_partition():
ledger = empty_ledger()
assert "candidate_facts" in ledger and ledger["candidate_facts"] == []
assert "confirmed_facts" in ledger
# ── P1-3: approval ledger integrity ────────────────────────────────────────
def test_pending_action_transitions_on_reject():
"""A pending action that is rejected moves to the failed/unknown partition
with status 'rejected' — it must not stay in pending and must not appear in
executed."""
ledger = empty_ledger()
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
status="pending", agent_id="seat-1")
assert len(ledger["pending_actions"]) == 1
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
status="rejected", agent_id="seat-1")
# Removed from pending.
assert not any(p.get("action_id") == "act-1" for p in ledger["pending_actions"])
# Present in failed/unknown with status rejected.
failed = [f for f in ledger["failed_or_unknown_actions"] if f.get("action_id") == "act-1"]
assert len(failed) == 1
assert failed[0]["status"] == "rejected"
# NOT in executed.
assert not any(e.get("action_id") == "act-1" for e in ledger["executed_actions"])
def test_pending_action_transitions_on_execute():
"""A pending action that executes successfully moves pending → executed."""
ledger = empty_ledger()
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
status="pending", agent_id="seat-1")
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
status="executed", agent_id="seat-1")
assert not any(p.get("action_id") == "act-2" for p in ledger["pending_actions"])
assert any(e.get("action_id") == "act-2" and e.get("status") == "executed"
for e in ledger["executed_actions"])
assert not any(f.get("action_id") == "act-2" for f in ledger["failed_or_unknown_actions"])
def test_failed_action_never_in_executed():
"""A failed/unknown action must never leak into executed_actions."""
ledger = empty_ledger()
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
status="pending", agent_id="seat-1")
for terminal in ("failed", "unknown"):
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
status=terminal, agent_id="seat-1")
assert not any(e.get("action_id") == "act-3" for e in ledger["executed_actions"]), \
f"{terminal} action leaked into executed_actions"
# ── P1-1: CrossReviewOutput parsing ────────────────────────────────────────
def test_parse_cross_review_output_valid():
payload = {
"summary": "审查通过",
"evidence_ok": True,
"gaps": ["缺少 X 证据"],
"conflicts": [],
"tool_failures": [],
"write_status_notes": ["待审批1个"],
"recommendations": ["补充X"],
"confirmed_candidate_ids": ["c1", "c2"],
}
model, errors = parse_cross_review_output(json.dumps(payload))
assert model is not None
assert errors == []
assert model.summary == "审查通过"
assert model.confirmed_candidate_ids == ["c1", "c2"]
def test_parse_cross_review_confirmed_candidates_requires_evidence():
"""Phase-3 P1-1: each ConfirmedCandidate must carry ≥1 evidence ref — an
empty list is a schema violation (so the executor repairs instead of
silently accepting a bare-id confirmation)."""
payload = {
"summary": "ok",
"confirmed_candidates": [{"candidate_id": "c1", "evidence_refs": []}],
}
model, errors = parse_cross_review_output(json.dumps(payload))
assert model is None
assert any("evidence_refs" in e or "confirmed_candidates" in e for e in errors)
def test_parse_cross_review_evidence_ok_defaults_false():
"""Phase-3 P1-1: omitting evidence_ok must default to False — a model that
forgets the field must never be treated as 'evidence sufficient'."""
model, errors = parse_cross_review_output(json.dumps({"summary": "ok"}))
assert model is not None
assert errors == []
assert model.evidence_ok is False
def test_parse_cross_review_output_invalid_json_returns_errors():
model, errors = parse_cross_review_output("not json at all")
assert model is None
assert errors # non-empty readable errors
def test_parse_cross_review_output_missing_summary_invalid():
"""summary is required (min_length=1) — a missing summary must fail
validation so the CrossReviewExecutor repairs rather than accepting it."""
model, errors = parse_cross_review_output(json.dumps({"gaps": ["x"]}))
assert model is None
assert any("summary" in e for e in errors)
# ── P1-1: CrossReviewExecutor deterministic degradation ────────────────────
#
# These exercise the executor's _apply_cross_review_output path with a fake
# adapter, proving: (a) valid output confirms candidates; (b) invalid output
# that the repair also cannot fix degrades deterministically WITHOUT pasting
# raw JSON into summary.
def _make_state():
"""Minimal RoundtableState for CrossReviewExecutor tests."""
from app.maf_roundtable.workflow_executors import RoundtableState
return RoundtableState(
session_id="s1", run_id="r1", task_text="t", mode="structured",
model_name=None, reasoning_effort=None, max_rounds=8,
write_policy="confirm_high_risk", manager_agent_id="mgr",
seat_agent_ids=["seat-1"], thread_map={"mgr": "t-mgr"},
)
@pytest.mark.asyncio
async def test_cross_review_valid_output_confirms_candidates():
from app.maf_roundtable.fact_ledger import record_tool_result
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
# Record evidence so the confirmation can bind it (Phase-3 P1-1).
record_tool_result(
state.fact_ledger, tool_name="search", status="completed",
agent_id="seat-1", turn_index=1,
)
tool_ref = "tool:search:1"
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
review: dict = {}
# Structured confirmed_candidates path: each candidate cites ledger evidence.
raw = json.dumps({
"summary": "ok",
"evidence_ok": True,
"confirmed_candidates": [{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}],
})
await executor._apply_cross_review_output(review, raw, "mgr")
assert review["source"] == "manager"
assert review["schema_status"] == "ok"
assert review["summary"] == "ok"
# The candidate was promoted to confirmed_facts.
assert len(state.fact_ledger["confirmed_facts"]) == 1
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
# The public confirmation field contains only the server-accepted item;
# the model request is retained separately for audit.
assert review["confirmed_candidate_ids"] == [candidate_id]
assert review["confirmed_candidates"] == [
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
]
assert review["requested_confirmations"] == review["confirmed_candidates"]
@pytest.mark.asyncio
async def test_cross_review_rejects_candidate_with_invented_evidence():
"""Phase-3 P1-1: a confirmation that cites evidence NOT in the ledger is
rejected server-side. The candidate stays unconfirmed, and the rejection is
surfaced in the review so the summary can explain it."""
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
review: dict = {}
raw = json.dumps({
"summary": "ok",
"evidence_ok": True,
"confirmed_candidates": [
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]},
],
})
await executor._apply_cross_review_output(review, raw, "mgr")
# Not promoted.
assert state.fact_ledger["confirmed_facts"] == []
# Rejection recorded.
assert review.get("confirmation_rejections"), "rejection not surfaced in review"
rej = review["confirmation_rejections"][0]
assert rej["candidate_id"] == candidate_id
assert rej["reason"] == "evidence_not_in_ledger"
assert review["confirmed_candidates"] == []
assert review["confirmed_candidate_ids"] == []
assert review["requested_confirmations"] == [
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]}
]
@pytest.mark.asyncio
async def test_cross_review_invalid_then_repair_succeeds():
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
# Fake adapter: repair turn returns valid JSON.
good_repair = json.dumps({"summary": "repaired ok", "confirmed_candidate_ids": []})
fake_adapter = SimpleNamespace(
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content=good_repair)),
)
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
review: dict = {}
# First output is garbage; repair returns valid.
await executor._apply_cross_review_output(review, "totally not json", "mgr")
assert review["source"] == "manager"
assert review["schema_status"] == "repaired"
assert review["summary"] == "repaired ok"
fake_adapter.run_seat_turn.assert_awaited_once()
@pytest.mark.asyncio
async def test_cross_review_invalid_and_repair_fails_degrades_deterministically():
"""When both the original and the repair output are unparseable, the
review MUST degrade deterministically: raw JSON is never pasted into
summary, and the review is marked schema_invalid."""
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
# Repair also returns garbage.
fake_adapter = SimpleNamespace(
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content="still garbage")),
)
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
review: dict = {}
await executor._apply_cross_review_output(review, "garbage first output", "mgr")
assert review["source"] == "deterministic"
assert review["schema_status"] == "schema_invalid"
assert review.get("raw_response_ignored") is True
# summary must NOT be the raw model JSON.
assert review.get("summary", "") != "garbage first output"
# ── P1-2: evidence_ok=False blocks all promotion ────────────────────────────
@pytest.mark.asyncio
async def test_evidence_ok_false_blocks_promotion_even_with_valid_refs():
"""Phase-3 P1-2: when the model says evidence_ok=False, no candidate may
be promoted even if it cites valid ledger-backed evidence refs. The
attempted confirmations are recorded as rejections."""
from app.maf_roundtable.fact_ledger import record_tool_result
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
record_tool_result(
state.fact_ledger, tool_name="search", status="completed",
agent_id="seat-1", turn_index=1,
)
tool_ref = "tool:search:1"
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
review: dict = {}
raw = json.dumps({
"summary": "证据不充分",
"evidence_ok": False,
"confirmed_candidates": [
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
],
})
await executor._apply_cross_review_output(review, raw, "mgr")
assert state.fact_ledger["confirmed_facts"] == []
assert candidate_id_map(state.fact_ledger)[candidate_id]["confirmed"] is False
assert review.get("confirmation_rejections")
rej = review["confirmation_rejections"][0]
assert rej["candidate_id"] == candidate_id
assert rej["reason"] == "review_did_not_confirm_evidence"
assert review["confirmed_candidates"] == []
assert review["confirmed_candidate_ids"] == []
assert review["requested_confirmations"] == [
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
]
@pytest.mark.asyncio
async def test_evidence_ok_true_allows_promotion_with_valid_refs():
"""Phase-3 P1-2: when evidence_ok=True AND evidence refs are valid, the
candidate IS promoted."""
from app.maf_roundtable.fact_ledger import record_tool_result
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
state = _make_state()
merge_seat_output(state.fact_ledger, {"findings": ["事实B"]}, agent_id="seat-1")
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
record_tool_result(
state.fact_ledger, tool_name="search", status="completed",
agent_id="seat-1", turn_index=1,
)
tool_ref = "tool:search:1"
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
review: dict = {}
raw = json.dumps({
"summary": "证据充分",
"evidence_ok": True,
"confirmed_candidates": [
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
],
})
await executor._apply_cross_review_output(review, raw, "mgr")
assert len(state.fact_ledger["confirmed_facts"]) == 1
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
@@ -0,0 +1,619 @@
"""Phase-3 Round-2 acceptance tests (P0-1): race-free idempotency & replay.
These prove the Phase-3 promise: a distinct DeerFlow execution (run, agent,
turn, execution_stage) is started at most once, even under concurrency,
retries, recovery, or a partial crash. They use a REAL SQLite-backed store +
the REAL ``DeerFlowAgentAdapter`` (only ``_execute_run`` is faked, so the
adapter's get_or_create / status-branching / snapshot logic is exercised
end-to-end against a real DB).
Coverage:
- Two coroutines acquiring the same four-tuple concurrently → exactly one wins
(created=True), the other reads the existing row (created=False). This is the
race that "query-then-insert" lost; the atomic insert-then-read wins it.
- A double-call of ``run_seat_turn`` after completion does NOT start a second
DeerFlow run — it replays the saved ``result_snapshot``.
- A call while the stage is pending/running returns an in-flight marker and
does NOT start a second DeerFlow run.
- A failed/cancelled stage is terminal and never auto-retried.
"""
from __future__ import annotations
import asyncio
import importlib.util
import logging
from datetime import datetime
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
from maf_test_helpers import make_maf_service
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter, SeatResponse
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
logger = logging.getLogger(__name__)
@pytest_asyncio.fixture
async def svc(tmp_path):
# IMPORTANT: a file-based SQLite is required for the concurrency test. The
# in-memory engine (``sqlite+aiosqlite://``) multiplexes all sessions over a
# SINGLE connection, so two coroutines in ``asyncio.gather`` share one
# transaction — the loser's rollback would void the winner's uncommitted
# insert, making a true race impossible to simulate. A file DB gives each
# ``async with self._sf()`` its own connection (production behavior), so the
# four-column unique constraint is the real arbiter.
db_path = tmp_path / "idempotency.db"
engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = make_maf_roundtable_store(sf)
service = make_maf_service(store)
# Seed a session + run so the seat_runs FK is satisfiable.
now = datetime(2026, 1, 1)
async with sf() as session:
session.add(MafRoundtableSessionRow(
id="sess-1", user_id="u1", title="t", task_text="t",
mode="structured", status="running", write_policy="auto",
max_rounds=8, created_at=now, updated_at=now,
))
session.add(MafRoundtableRunRow(
id="run-1", session_id="sess-1", status="running",
created_at=now, updated_at=now,
))
await session.commit()
yield service
await engine.dispose()
def _adapter(svc) -> DeerFlowAgentAdapter:
app = MagicMock()
app.state.checkpointer = AsyncMock()
app.state.thread_store = AsyncMock()
app.state.stream_bridge = None
app.state.run_manager = AsyncMock()
client = DeerFlowRuntimeClient(app)
return DeerFlowAgentAdapter(client, service=svc, user_id="u1")
# ── 1. Concurrent acquisition: exactly one wins ─────────────────────────────
@pytest.mark.asyncio
async def test_concurrent_get_or_create_one_winner(svc):
"""Two coroutines acquire the same four-tuple at (near-)the same time. The
atomic insert-then-read means the DB constraint picks a single winner; the
loser reads the winner's row. Both calls return, neither raises, and there
is exactly ONE row for the four-tuple."""
repo = svc.store.seat_runs
key = dict(maf_run_id="run-1", agent_id="a1", turn_index=1, execution_stage="expert")
(ar, ac), (br, bc) = await asyncio.gather(
repo.get_or_create_seat_run(**key),
repo.get_or_create_seat_run(**key),
)
assert (ac, bc) in {(True, False), (False, True)}, "exactly one winner; the other reads existing"
assert ar["id"] == br["id"]
rows = await repo.get_by_maf_run("run-1")
assert len([r for r in rows if r["agent_id"] == "a1" and r["execution_stage"] == "expert"]) == 1
# ── 2. Double-call after completion replays the snapshot ────────────────────
@pytest.mark.asyncio
async def test_double_call_after_completion_does_not_re_run(svc):
"""A second ``run_seat_turn`` for an already-completed stage must replay the
saved ``result_snapshot`` and NOT call ``_execute_run`` again. This is the
core recovery guarantee: a restart or retry never re-invokes the model."""
adapter = _adapter(svc)
calls = {"execute": 0}
async def _fake_execute(**kwargs):
calls["execute"] += 1
return SeatResponse(
agent_id=kwargs["agent_id"],
turn_index=kwargs["turn_index"],
content="first-run-output",
tool_calls=[{"name": "search", "args": {}, "id": "tc1"}],
usage={"input_tokens": 5, "output_tokens": 5},
metadata={"thread_id": kwargs["thread_id"]},
)
adapter._execute_run = _fake_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
common = dict(
session_id="sess-1", run_id="run-1", agent_id="a2", turn_index=2,
execution_stage="expert", message="q", thread_id="t-a2",
task_text="q", stage_instruction="",
)
first = await adapter.run_seat_turn(**common)
assert calls["execute"] == 1
assert first.content == "first-run-output"
# Second call — completed stage. Must replay, NOT re-run.
second = await adapter.run_seat_turn(**common)
assert calls["execute"] == 1, "completed stage was re-run instead of replayed"
assert second.content == "first-run-output"
assert second.error is None
# ── 3. In-flight stage: second caller waits and replays ─────────────────────
@pytest.mark.asyncio
async def test_running_stage_second_caller_waits_and_replays(svc):
"""When a stage is pending/running (another caller owns it), a second
``run_seat_turn`` must WAIT for the first to complete, then replay the
same snapshot. ``_execute_run`` is called exactly once."""
adapter = _adapter(svc)
execute_started = asyncio.Event()
release_execute = asyncio.Event()
calls = {"execute": 0}
async def _blocking_execute(**kwargs):
calls["execute"] += 1
execute_started.set()
await release_execute.wait()
return SeatResponse(
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
content="done-content", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
usage={"input_tokens": 1}, metadata={"thread_id": kwargs["thread_id"]},
)
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
common = dict(
session_id="sess-1", run_id="run-1", agent_id="a3", turn_index=3,
execution_stage="cross_review", message="q", thread_id="t-a3",
task_text="q", stage_instruction="",
)
first_task = asyncio.create_task(adapter.run_seat_turn(**common))
await execute_started.wait()
second_task = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
await asyncio.sleep(0.05)
release_execute.set()
first = await first_task
second = await second_task
assert calls["execute"] == 1, "in-flight stage started a second DeerFlow run"
assert first.content == "done-content"
assert second.content == "done-content"
assert second.error is None
# ── 4. Failed/cancelled stage is terminal ────────────────────────────────────
@pytest.mark.asyncio
async def test_failed_stage_not_auto_retried(svc):
"""A previously-failed stage is terminal; a new ``run_seat_turn`` surfaces
a terminal marker and never auto-retries (no silent re-run)."""
adapter = _adapter(svc)
calls = {"execute": 0}
async def _failing_execute(**kwargs):
calls["execute"] += 1
raise RuntimeError("model blew up")
adapter._execute_run = _failing_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
common = dict(
session_id="sess-1", run_id="run-1", agent_id="a4", turn_index=4,
execution_stage="finalizer", message="q", thread_id="t-a4",
task_text="q", stage_instruction="",
)
first = await adapter.run_seat_turn(**common)
assert first.error # failed
assert calls["execute"] == 1
second = await adapter.run_seat_turn(**common)
assert calls["execute"] == 1, "failed stage was auto-retried"
assert second.error is not None
assert second.metadata is not None and second.metadata.get("terminal") == "failed"
# ── 5. Four-tuple replay snapshot persists sanitized content ─────────────────
@pytest.mark.asyncio
async def test_result_snapshot_persisted_on_completion(svc):
"""The sanitized result snapshot is written atomically with the completed
transition so recovery can replay it even if the workflow checkpoint was
not yet written."""
adapter = _adapter(svc)
async def _fake_execute(**kwargs):
return SeatResponse(
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
content="snapshot-me", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
usage={"input_tokens": 1}, metadata={"thread_id": "t"},
)
adapter._execute_run = _fake_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
await adapter.run_seat_turn(
session_id="sess-1", run_id="run-1", agent_id="a5", turn_index=5,
execution_stage="expert", message="q", thread_id="t-a5",
task_text="q", stage_instruction="",
)
row = await svc.store.seat_runs.get_by_four_tuple(
maf_run_id="run-1", agent_id="a5", turn_index=5, execution_stage="expert",
)
assert row is not None
assert row["status"] == "completed"
snap = row["result_snapshot"]
assert snap is not None
assert snap["content"] == "snapshot-me"
assert snap["tool_calls"][0]["name"] == "t"
# ── 6. Invalid execution_stage rejected ──────────────────────────────────────
@pytest.mark.asyncio
async def test_invalid_execution_stage_rejected(svc):
"""A typo'd execution_stage (e.g. 'exprt') is rejected at the repository —
it would otherwise bypass the four-column idempotency key by creating a
brand-new stage."""
from deerflow.persistence.maf_roundtable.seat_runs import InvalidExecutionStageError
repo = svc.store.seat_runs
with pytest.raises(InvalidExecutionStageError):
await repo.get_or_create_seat_run(
maf_run_id="run-1", agent_id="a6", turn_index=6,
execution_stage="exprt", # typo
)
with pytest.raises(InvalidExecutionStageError):
await repo.create_seat_run(
maf_run_id="run-1", agent_id="a6", turn_index=6,
execution_stage="expert2", # not in the closed enum
)
# ── 7. Migration branching: constraint vs index drop ─────────────────────────
def _load_phase3_migration():
here = Path(__file__).resolve().parent.parent
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
assert spec and spec.loader, "could not load phase3 migration script"
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def test_migration_ddl_drop_branches_on_kind():
"""The migration must emit DROP CONSTRAINT for a table constraint and DROP
INDEX for a unique index — the Phase-2 deployment created the constraint as
a CREATE UNIQUE INDEX, so calling DROP CONSTRAINT on it failed."""
mod = _load_phase3_migration()
# A constraint-kind object → DROP CONSTRAINT.
drop_constraint_sql = mod._ddl_drop_constraint("postgresql", "uq_maf_seat_runs_run_agent_turn")
assert "DROP CONSTRAINT" in drop_constraint_sql
# An index-kind object → DROP INDEX.
drop_index_sql = mod._ddl_drop_index("uq_maf_seat_runs_run_agent_turn")
assert "DROP INDEX" in drop_index_sql
@pytest.mark.asyncio
async def test_migration_sqlite_rebuild_restores_deerflow_run_id_index():
"""The SQLite rebuild must recreate the ``deerflow_run_id`` index — without
it, cancel-by-deerflow-run-id lookups silently degrade after a restart."""
mod = _load_phase3_migration()
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
joined = "\n".join(rebuild)
assert "deerflow_run_id" in joined, "rebuild dropped the deerflow_run_id index"
assert "ck_maf_seat_runs_execution_stage" in joined
assert "execution_stage IN" in joined
# ── 8. P0-1: Two adapters same stage — wait, not partial_failure ─────────────
@pytest.mark.asyncio
async def test_two_adapters_same_stage_wait_not_partial_failure(svc):
"""Two adapter calls for the same four-tuple: the first blocks executing,
the second waits. After release, both get the same snapshot. _execute_run
is called exactly once. Neither sets partial_failure."""
adapter = _adapter(svc)
execute_started = asyncio.Event()
release_execute = asyncio.Event()
calls = {"execute": 0}
async def _blocking_execute(**kwargs):
calls["execute"] += 1
execute_started.set()
await release_execute.wait()
return SeatResponse(
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
content="shared-result", tool_calls=[], usage={"input_tokens": 2},
metadata={"thread_id": kwargs["thread_id"]},
)
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
common = dict(
session_id="sess-1", run_id="run-1", agent_id="a-wait", turn_index=7,
execution_stage="expert", message="q", thread_id="t-wait",
task_text="q", stage_instruction="",
)
task1 = asyncio.create_task(adapter.run_seat_turn(**common))
await execute_started.wait()
task2 = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
await asyncio.sleep(0.05)
release_execute.set()
r1, r2 = await asyncio.gather(task1, task2)
assert calls["execute"] == 1
assert r1.content == "shared-result"
assert r2.content == "shared-result"
assert r1.error is None
assert r2.error is None
# ── 9. P0-1: Empty snapshot defense ──────────────────────────────────────────
@pytest.mark.asyncio
async def test_completed_with_empty_snapshot_returns_error(svc):
"""A completed seat run with no result_snapshot must NOT produce a fake
success — it returns an explicit error requiring manual recovery."""
adapter = _adapter(svc)
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
seat_run = await svc.store.seat_runs.create_seat_run(
maf_run_id="run-1", agent_id="a-empty", turn_index=8, execution_stage="expert",
)
await svc.store.seat_runs.mark_completed(seat_run["id"], result_snapshot=None)
result = await adapter.run_seat_turn(
session_id="sess-1", run_id="run-1", agent_id="a-empty", turn_index=8,
execution_stage="expert", message="q", thread_id="t-empty",
task_text="q", stage_instruction="",
)
assert result.error is not None
assert "no replayable snapshot" in result.error
assert result.content == ""
# ── 10. P0-2: Snapshot sanitization before persist ───────────────────────────
@pytest.mark.asyncio
async def test_result_snapshot_sanitized_before_persist(svc):
"""Secrets in tool_calls args and metadata must be redacted in the
persisted snapshot. Safe fields are preserved."""
adapter = _adapter(svc)
async def _fake_execute(**kwargs):
return SeatResponse(
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
content="analysis complete",
tool_calls=[{
"name": "request_api",
"args": {
"api_key": "secret-api-key",
"password": "secret-password",
"normal": "safe",
},
"id": "tc1",
}],
usage={"input_tokens": 1},
metadata={"authorization": "Bearer secret-token", "thread_id": "t"},
)
adapter._execute_run = _fake_execute # type: ignore[method-assign]
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
adapter._emit = AsyncMock() # type: ignore[method-assign]
await adapter.run_seat_turn(
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
execution_stage="expert", message="q", thread_id="t-san",
task_text="q", stage_instruction="",
)
row = await svc.store.seat_runs.get_by_four_tuple(
maf_run_id="run-1", agent_id="a-san", turn_index=9, execution_stage="expert",
)
assert row is not None
snap = row["result_snapshot"]
assert snap is not None
snap_str = str(snap)
assert "secret-api-key" not in snap_str
assert "secret-password" not in snap_str
assert "secret-token" not in snap_str
assert snap["tool_calls"][0]["args"]["normal"] == "safe"
replayed = await adapter.run_seat_turn(
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
execution_stage="expert", message="q", thread_id="t-san",
task_text="q", stage_instruction="",
)
assert replayed.content == "analysis complete"
assert "secret-api-key" not in str(replayed.tool_calls)
# ── 11. P1-1: Migration intermediate state tests ─────────────────────────────
def test_migration_rebuild_preserves_existing_stage_and_snapshot():
"""When the source table already has execution_stage and result_snapshot,
the rebuild must copy them verbatim — not overwrite with 'legacy'/NULL."""
mod = _load_phase3_migration()
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=True, source_has_snapshot=True)
insert_sql = rebuild[1]
assert "execution_stage" in insert_sql
assert "'legacy'" not in insert_sql.split("SELECT")[1]
assert "result_snapshot" in insert_sql
assert "NULL" not in insert_sql.split("SELECT")[1].split("FROM")[0].replace("result_snapshot", "")
def test_migration_rebuild_from_phase2_uses_defaults():
"""When the source table has neither stage nor snapshot (Phase-2), the
rebuild uses 'legacy' and NULL."""
mod = _load_phase3_migration()
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
insert_sql = rebuild[1]
select_part = insert_sql.split("SELECT")[1].split("FROM")[0]
assert "'legacy'" in select_part
assert "NULL" in select_part
@pytest.mark.asyncio
async def test_migration_idempotent_double_run(tmp_path):
"""Running the migration twice on the same SQLite DB succeeds both times."""
import sqlite3
db_path = tmp_path / "double_migrate.db"
conn = sqlite3.connect(str(db_path))
conn.execute("""
CREATE TABLE maf_roundtable_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
session_id VARCHAR(64) NOT NULL,
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
)
""")
conn.execute("""
CREATE TABLE maf_roundtable_seat_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
maf_run_id VARCHAR(64) NOT NULL,
agent_id VARCHAR(64) NOT NULL,
turn_index INTEGER NOT NULL,
deerflow_run_id VARCHAR(64),
deerflow_thread_id VARCHAR(64),
sandbox_id VARCHAR(64),
workflow_checkpoint_id VARCHAR(128),
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
started_at DATETIME,
finished_at DATETIME,
error TEXT,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
CONSTRAINT uq_maf_seat_runs_run_agent_turn UNIQUE (maf_run_id, agent_id, turn_index)
)
""")
conn.execute("""
INSERT INTO maf_roundtable_seat_runs
(id, maf_run_id, agent_id, turn_index, status, created_at, updated_at)
VALUES ('sr1', 'run-1', 'a1', 1, 'completed', '2026-01-01', '2026-01-01')
""")
conn.commit()
conn.close()
from sqlalchemy import create_engine, inspect, text
engine = create_engine(f"sqlite:///{db_path}")
mod = _load_phase3_migration()
for _ in range(2):
with engine.connect() as sync_conn:
mod._do_migrate_sync(
sync_conn, dry_run=False, logger=logger, changes_ref=[0]
)
sync_conn.commit()
insp = inspect(engine)
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
assert "execution_stage" in cols
assert "result_snapshot" in cols
with engine.connect() as conn:
row = conn.execute(text("SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr1'")).fetchone()
assert row[0] == "legacy"
engine.dispose()
@pytest.mark.asyncio
async def test_migration_intermediate_preserves_expert_stage(tmp_path):
"""An intermediate table (has stage + 4-col unique, but no snapshot/check)
must preserve existing execution_stage='expert' and result_snapshot data
after rebuild."""
import sqlite3
db_path = tmp_path / "intermediate.db"
conn = sqlite3.connect(str(db_path))
conn.execute("""
CREATE TABLE maf_roundtable_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
session_id VARCHAR(64) NOT NULL,
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
)
""")
conn.execute("""
CREATE TABLE maf_roundtable_seat_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
maf_run_id VARCHAR(64) NOT NULL,
agent_id VARCHAR(64) NOT NULL,
turn_index INTEGER NOT NULL,
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
deerflow_run_id VARCHAR(64),
deerflow_thread_id VARCHAR(64),
sandbox_id VARCHAR(64),
workflow_checkpoint_id VARCHAR(128),
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
result_snapshot TEXT,
started_at DATETIME,
finished_at DATETIME,
error TEXT,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
CONSTRAINT uq_maf_seat_runs_run_agent_turn_stage UNIQUE (maf_run_id, agent_id, turn_index, execution_stage)
)
""")
conn.execute("""
INSERT INTO maf_roundtable_seat_runs
(id, maf_run_id, agent_id, turn_index, execution_stage, status, result_snapshot, created_at, updated_at)
VALUES ('sr2', 'run-1', 'a1', 1, 'expert', 'completed', '{"content": "keep-me"}', '2026-01-01', '2026-01-01')
""")
conn.commit()
conn.close()
from sqlalchemy import create_engine, text
engine = create_engine(f"sqlite:///{db_path}")
mod = _load_phase3_migration()
with engine.connect() as sync_conn:
mod._do_migrate_sync(sync_conn, dry_run=False, logger=logger, changes_ref=[0])
sync_conn.commit()
with engine.connect() as conn:
row = conn.execute(text(
"SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr2'"
)).fetchone()
assert row[0] == "expert", f"stage corrupted to {row[0]!r}"
assert "keep-me" in (row[1] or ""), f"snapshot corrupted to {row[1]!r}"
engine.dispose()
@@ -0,0 +1,264 @@
"""Phase-3 Round-2 acceptance tests (P0-3): real manager workflow coexistence.
Drives the REAL workflow executors (PlannerExecutor → SeatExecutor ×N →
CrossReviewExecutor → ApprovalGateExecutor → FinalizerExecutor) through a
manager + 2-seat structured run, with a fake DeerFlow adapter whose
``run_seat_turn`` records each call's ``execution_stage`` into a REAL seat-run
store. This proves the Phase-3 disambiguation invariants end-to-end:
- manager planner / cross_review / finalizer all share the SAME business round
(turn_index) but occupy distinct seat-run rows via ``execution_stage`` — they
must coexist without a four-tuple collision.
- an expert seat turn and its one-shot ``format_repair`` retry coexist for the
same (agent, turn).
- invalid expert output triggers exactly one ``format_repair`` DeerFlow call,
and both rows persist.
- invalid cross-review output triggers exactly one ``cross_review_repair``
DeerFlow call, and ``cross_review`` + ``cross_review_repair`` rows coexist.
"""
from __future__ import annotations
import json
from datetime import datetime
from unittest.mock import AsyncMock
import pytest
import pytest_asyncio
from maf_test_helpers import make_maf_service
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.maf_roundtable.deerflow_agent_adapter import SeatResponse
from app.maf_roundtable.workflow_executors import (
ApprovalGateExecutor,
CrossReviewExecutor,
FinalizerExecutor,
PlannerExecutor,
RoundtableState,
SeatExecutor,
)
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
MANAGER = "manager-1"
SEATS = ["seat-a", "seat-b"]
class _FakeCtx:
def __init__(self) -> None:
self.messages: list[object] = []
self.outputs: list[object] = []
async def send_message(self, msg: object) -> None:
self.messages.append(msg)
async def yield_output(self, msg: object) -> None:
self.outputs.append(msg)
@pytest_asyncio.fixture
async def svc():
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = make_maf_roundtable_store(sf)
service = make_maf_service(store)
now = datetime(2026, 1, 1)
async with sf() as session:
session.add(MafRoundtableSessionRow(
id="sess-1", user_id="u1", title="t", task_text="分析市场",
mode="structured", status="running", write_policy="auto",
manager_agent_id=MANAGER, max_rounds=8, created_at=now, updated_at=now,
))
session.add(MafRoundtableRunRow(
id="run-1", session_id="sess-1", status="running",
created_at=now, updated_at=now,
))
await session.commit()
yield service
await engine.dispose()
def _state() -> RoundtableState:
return RoundtableState(
session_id="sess-1", run_id="run-1", task_text="分析市场",
mode="structured", model_name=None, reasoning_effort=None,
max_rounds=8, write_policy="auto", manager_agent_id=MANAGER,
seat_agent_ids=SEATS,
thread_map={MANAGER: "t-mgr", "seat-a": "t-a", "seat-b": "t-b"},
capability_snapshot=None, seat_timeout_seconds=5, schema_repair_enabled=True,
)
def _resp(content: str) -> SeatResponse:
return SeatResponse(
agent_id="x", turn_index=0, content=content,
tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1},
)
class _RecordingAdapter:
"""Fake adapter that (a) records every run_seat_turn's execution_stage and
(b) writes a REAL seat-run row for each call so the four-tuple coexistence
invariant is checked against the actual DB."""
def __init__(self, service) -> None:
self._service = service
self.calls: list[tuple[str, str, int, str]] = [] # (agent, turn, stage)
self._seat_outputs: dict[tuple[str, str], str] = {} # (agent, stage) -> content
self._user_id = "u1"
self._emit_critical = AsyncMock()
self._emit = AsyncMock()
def set_output(self, agent_id: str, execution_stage: str, content: str) -> None:
self._seat_outputs[(agent_id, execution_stage)] = content
async def run_seat_turn(self, **kwargs):
agent_id = kwargs["agent_id"]
turn_index = kwargs["turn_index"]
stage = kwargs.get("execution_stage", "expert")
self.calls.append((agent_id, turn_index, stage))
# Persist a seat-run row so coexistence is verified against the real DB.
await self._service.store.seat_runs.create_seat_run(
maf_run_id=kwargs["run_id"], agent_id=agent_id,
turn_index=turn_index, execution_stage=stage,
)
content = self._seat_outputs.get((agent_id, stage), '{"summary":"ok"}')
return _resp(content)
# ── 1. Full structured run: manager stages coexist ───────────────────────────
@pytest.mark.asyncio
async def test_full_structured_run_manager_stages_coexist(svc):
"""A complete planner→experts→cross_review→approval→finalizer run must
persist manager planner/cross_review/finalizer rows that ALL share the same
business round (turn_index) but differ by execution_stage — no collision."""
adapter = _RecordingAdapter(svc)
# Manager emits valid JSON at each stage.
adapter.set_output(MANAGER, "planner", "分工:seat-a 负责需求,seat-b 负责竞品")
adapter.set_output(MANAGER, "cross_review", json.dumps({
"summary": "审查通过",
"evidence_ok": True,
"confirmed_candidates": [],
}))
adapter.set_output(MANAGER, "finalizer", "最终结论:市场看好")
# Seats emit valid structured output.
for s in SEATS:
adapter.set_output(s, "expert", json.dumps({"summary": f"{s} 结论", "findings": ["f"]}))
state = _state()
ctx = _FakeCtx()
await PlannerExecutor(adapter, state).handle("task", ctx)
for s in SEATS:
await SeatExecutor(adapter, state, s).handle({}, _FakeCtx())
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
await ApprovalGateExecutor(adapter, state, svc).handle({}, _FakeCtx())
await FinalizerExecutor(adapter, state).handle({}, _FakeCtx())
# Assert every manager stage produced a distinct seat-run row, all sharing
# turn_index (planner=0, cross_review/finalizer advance current_round; the
# invariant is that the SAME (agent) has multiple distinct stages).
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
mgr_rows = [r for r in rows if r["agent_id"] == MANAGER]
mgr_stages = {r["execution_stage"] for r in mgr_rows}
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
f"manager stages missing; got {mgr_stages}"
# No duplicate four-tuple (the unique constraint would have raised).
seen = set()
for r in mgr_rows:
key = (r["maf_run_id"], r["agent_id"], r["turn_index"], r["execution_stage"])
assert key not in seen, f"duplicate four-tuple {key}"
seen.add(key)
# The final result was produced.
assert state.final_result is not None
assert state.status == "completed"
# ── 2. expert + format_repair coexist ────────────────────────────────────────
@pytest.mark.asyncio
async def test_expert_format_repair_coexist(svc):
"""A seat that emits invalid JSON triggers exactly one ``format_repair``
DeerFlow call; both the expert and the format_repair rows persist for the
same (agent, turn) without colliding."""
adapter = _RecordingAdapter(svc)
# First expert call: invalid JSON → triggers repair. The format_repair call
# (stage='format_repair') returns valid JSON.
adapter.set_output("seat-a", "expert", "not-json-at-all")
adapter.set_output("seat-a", "format_repair", json.dumps({"summary": "repaired"}))
state = _state()
state.current_round = 1
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
stage_calls = [(a, t, s) for (a, t, s) in adapter.calls if a == "seat-a"]
stages = [s for _, _, s in stage_calls]
assert "expert" in stages and "format_repair" in stages
# Exactly one repair (not a retry loop).
assert stages.count("format_repair") == 1
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
seat_a_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "seat-a"}
assert seat_a_stages == {"expert", "format_repair"}
# ── 3. cross_review + cross_review_repair coexist ────────────────────────────
@pytest.mark.asyncio
async def test_cross_review_repair_coexist(svc):
"""An invalid cross-review output triggers exactly one ``cross_review_repair``
DeerFlow call; cross_review and cross_review_repair rows coexist for the
manager."""
adapter = _RecordingAdapter(svc)
adapter.set_output(MANAGER, "cross_review", "totally not json")
adapter.set_output(MANAGER, "cross_review_repair", json.dumps({
"summary": "repaired review", "confirmed_candidates": [],
}))
state = _state()
# Pretend experts already ran so the cross-review barrier sees outcomes.
state.seat_outcomes = {s: {"status": "completed"} for s in SEATS}
state.current_round = 1
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
mgr_stages = [s for a, _, s in adapter.calls if a == MANAGER]
assert "cross_review" in mgr_stages and "cross_review_repair" in mgr_stages
assert mgr_stages.count("cross_review_repair") == 1
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
mgr_row_stages = {r["execution_stage"] for r in rows if r["agent_id"] == MANAGER}
assert {"cross_review", "cross_review_repair"}.issubset(mgr_row_stages)
# The repaired review was applied (deterministic degradation did NOT happen).
assert state.cross_review_result is not None
assert state.cross_review_result.get("schema_status") == "repaired"
assert state.cross_review_result.get("summary") == "repaired review"
# ── 4. manager present in participants-style config ──────────────────────────
@pytest.mark.asyncio
async def test_manager_role_wired_through_state(svc):
"""Sanity: RoundtableState carries manager_agent_id and the planner uses it
for the planner seat-turn (not a seat id)."""
state = _state()
assert state.manager_agent_id == MANAGER
assert MANAGER not in state.seat_agent_ids
adapter = _RecordingAdapter(svc)
adapter.set_output(MANAGER, "planner", "plan")
ctx = _FakeCtx()
await PlannerExecutor(adapter, state).handle("task", ctx)
planner_calls = [(a, s) for a, _, s in adapter.calls if s == "planner"]
assert planner_calls == [(MANAGER, "planner")], \
"planner stage must run as the MANAGER, not a seat"
@@ -0,0 +1,318 @@
"""Phase-3 acceptance tests (P0-2): real seat-run identity & idempotency.
These tests use a REAL SQLite ``MafSeatRunRepository`` (no mocks on the
repository) to exercise the Phase-3 four-column unique constraint
``(maf_run_id, agent_id, turn_index, execution_stage)`` and the idempotent
``get_or_create_seat_run`` acquisition.
They prove the core Phase-3 principle:
The business round ``turn_index`` never changes, but every distinct
DeerFlow execution gets its own persistent identity via ``execution_stage``.
Coverage:
1. manager planner → cross_review → finalizer within ONE business round
coexist (same agent, same turn, different stage).
2. expert → format_repair within one business round coexist (same agent,
same turn, different stage).
3. The same four-tuple second insert is rejected by the DB constraint.
4. ``get_or_create_seat_run`` returns the existing row (created=False) instead
of launching a duplicate when one already exists.
5. Process recovery: an already-completed stage is NOT re-created.
6. Idempotent Phase-3 migration + old rows back-filled to ``legacy``.
"""
from __future__ import annotations
import importlib.util
from datetime import datetime
from pathlib import Path
import pytest
import pytest_asyncio
from sqlalchemy import inspect as sa_inspect
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
from deerflow.persistence.maf_roundtable.model import Base
from deerflow.persistence.maf_roundtable.seat_runs import MafSeatRunRepository
# ── fixtures ────────────────────────────────────────────────────────────────
@pytest_asyncio.fixture
async def repo():
"""A real SQLite-backed MafSeatRunRepository over the full schema."""
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
# Seed the parent run row so the FK on seat_runs is satisfiable.
from deerflow.persistence.maf_roundtable.model import (
MafRoundtableRunRow,
MafRoundtableSessionRow,
)
now = datetime(2026, 1, 1)
async with sf() as session:
session.add(MafRoundtableSessionRow(
id="s1", user_id="u1", title="t", task_text="t",
mode="structured", status="draft", write_policy="confirm_high_risk",
max_rounds=8, created_at=now, updated_at=now,
))
session.add(MafRoundtableRunRow(
id="r1", session_id="s1", status="pending",
created_at=now, updated_at=now,
))
await session.commit()
r = MafSeatRunRepository(sf)
yield r
await engine.dispose()
# ── 1. Same business round, different stages coexist (manager path) ─────────
@pytest.mark.asyncio
async def test_manager_stages_coexist_within_one_business_round(repo):
"""planner / cross_review / finalizer all share (run, manager, turn) but
differ by execution_stage — all three must insert without conflict."""
common = dict(maf_run_id="r1", agent_id="manager-1", turn_index=1)
await repo.create_seat_run(execution_stage="planner", **common)
await repo.create_seat_run(execution_stage="cross_review", **common)
await repo.create_seat_run(execution_stage="finalizer", **common)
rows = await repo.get_by_maf_run("r1")
stages = {r["execution_stage"] for r in rows}
assert stages == {"planner", "cross_review", "finalizer"}
# All share the SAME business turn_index (never mutated to dodge collision).
assert {r["turn_index"] for r in rows} == {1}
# ── 2. expert + format_repair coexist within one business round ─────────────
@pytest.mark.asyncio
async def test_expert_and_format_repair_coexist(repo):
"""An expert seat turn and its one-shot format-repair retry share the
business round but are distinct DeerFlow executions."""
common = dict(maf_run_id="r1", agent_id="expert-1", turn_index=1)
await repo.create_seat_run(execution_stage="expert", **common)
# Same agent, same turn, DIFFERENT stage — must NOT conflict.
await repo.create_seat_run(execution_stage="format_repair", **common)
rows = await repo.get_by_maf_run("r1")
assert {r["execution_stage"] for r in rows} == {"expert", "format_repair"}
# ── 3. Same four-tuple second insert is rejected ────────────────────────────
@pytest.mark.asyncio
async def test_duplicate_four_tuple_rejected(repo):
"""Inserting the exact same (run, agent, turn, stage) twice must raise —
the four-column unique constraint enforces at-most-one DeerFlow execution
per stage per business round."""
common = dict(maf_run_id="r1", agent_id="a1", turn_index=2, execution_stage="expert")
await repo.create_seat_run(**common)
with pytest.raises(IntegrityError):
await repo.create_seat_run(**common)
# ── 4. get_or_create_seat_run is idempotent ─────────────────────────────────
@pytest.mark.asyncio
async def test_get_or_create_returns_existing_not_duplicate(repo):
"""When a seat-run for the four-tuple already exists, get_or_create must
return it with created=False rather than launching a duplicate."""
key = dict(maf_run_id="r1", agent_id="a2", turn_index=3, execution_stage="cross_review")
row1, created1 = await repo.get_or_create_seat_run(**key)
assert created1 is True
assert row1["execution_stage"] == "cross_review"
# Second acquisition — existing row returned, NO duplicate created.
row2, created2 = await repo.get_or_create_seat_run(**key)
assert created2 is False
assert row2["id"] == row1["id"]
rows = await repo.get_by_maf_run("r1")
cross_review_rows = [r for r in rows if r["agent_id"] == "a2"]
assert len(cross_review_rows) == 1 # exactly one, no duplicate
# ── 5. Process recovery: completed stage is NOT re-created ─────────────────
@pytest.mark.asyncio
async def test_recovery_completed_stage_not_recreated(repo):
"""After a simulated restart, acquiring the same four-tuple returns the
already-persisted row (created=False). The caller must replay the
checkpoint's saved stage result instead of starting a new DeerFlow run."""
key = dict(maf_run_id="r1", agent_id="a3", turn_index=4, execution_stage="finalizer")
row, created = await repo.get_or_create_seat_run(**key)
assert created is True
await repo.mark_completed(row["id"], checkpoint_id="ckpt-xyz")
# Simulated restart: fresh acquisition of the same stage.
row2, created2 = await repo.get_or_create_seat_run(**key)
assert created2 is False
assert row2["id"] == row["id"]
assert row2["status"] == "completed"
assert row2["workflow_checkpoint_id"] == "ckpt-xyz"
@pytest.mark.asyncio
async def test_get_by_four_tuple_lookup(repo):
"""Direct four-tuple lookup returns the row or None."""
key = dict(maf_run_id="r1", agent_id="a4", turn_index=5, execution_stage="expert")
assert await repo.get_by_four_tuple(**key) is None
await repo.create_seat_run(**key)
found = await repo.get_by_four_tuple(**key)
assert found is not None
assert found["agent_id"] == "a4"
assert found["execution_stage"] == "expert"
# ── 6. Phase-3 migration: idempotent + back-fills legacy ────────────────────
def _load_phase3_migration():
here = Path(__file__).resolve().parent.parent
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
assert spec and spec.loader, "could not load phase3 migration script"
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
@pytest.mark.asyncio
async def test_phase3_migration_idempotent_and_backfills_legacy():
"""Run the Phase-3 migration against a Phase-2 schema (old 3-tuple
constraint), then re-run it (idempotent). Old seat-run rows are
back-filled with execution_stage='legacy'."""
mod = _load_phase3_migration()
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
# Build the OLD Phase-2 seat_runs table (3-tuple unique, NO execution_stage).
async with engine.begin() as conn:
await conn.execute(text("""
CREATE TABLE maf_roundtable_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
session_id VARCHAR(64) NOT NULL,
status VARCHAR(32) NOT NULL,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
)
"""))
await conn.execute(text("""
CREATE TABLE maf_roundtable_sessions (
id VARCHAR(64) NOT NULL PRIMARY KEY,
user_id VARCHAR(64) NOT NULL,
title VARCHAR(255),
task_text TEXT,
mode VARCHAR(32),
status VARCHAR(32),
write_policy VARCHAR(32),
max_rounds INTEGER,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
)
"""))
await conn.execute(text("""
CREATE TABLE maf_roundtable_seat_runs (
id VARCHAR(64) NOT NULL PRIMARY KEY,
maf_run_id VARCHAR(64) NOT NULL,
agent_id VARCHAR(64) NOT NULL,
turn_index INTEGER NOT NULL,
deerflow_run_id VARCHAR(64),
deerflow_thread_id VARCHAR(64),
sandbox_id VARCHAR(64),
workflow_checkpoint_id VARCHAR(128),
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
started_at DATETIME,
finished_at DATETIME,
error TEXT,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE
)
"""))
await conn.execute(text(
"CREATE UNIQUE INDEX uq_maf_seat_runs_run_agent_turn "
"ON maf_roundtable_seat_runs (maf_run_id, agent_id, turn_index)"
))
# Seed parent run + a pre-Phase-3 seat-run row.
await conn.execute(text(
"INSERT INTO maf_roundtable_sessions (id, user_id, title, task_text, mode, status, write_policy, max_rounds, created_at, updated_at) "
"VALUES ('s1','u1','t','t','structured','draft','confirm_high_risk',8,'2026-01-01','2026-01-01')"
))
await conn.execute(text(
"INSERT INTO maf_roundtable_runs (id, session_id, status, created_at, updated_at) "
"VALUES ('r1','s1','pending','2026-01-01','2026-01-01')"
))
await conn.execute(text(
"INSERT INTO maf_roundtable_seat_runs (id, maf_run_id, agent_id, turn_index, status, created_at, updated_at) "
"VALUES ('sr1','r1','a1',1,'completed','2026-01-01','2026-01-01')"
))
# Run the migration.
async with engine.begin() as conn:
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
sf = async_sessionmaker(engine, expire_on_commit=False)
def _verify(sync_conn):
insp = sa_inspect(sync_conn)
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
assert "execution_stage" in cols
# The NEW four-column unique constraint/index is present.
new_constraint = mod._find_unique_constraint(
insp, "maf_roundtable_seat_runs",
expected_cols={"maf_run_id", "agent_id", "turn_index", "execution_stage"},
)
assert new_constraint is not None, "new four-column unique constraint missing"
# The OLD three-column constraint must be gone (rebuilt in SQLite).
old_constraint = mod._find_unique_constraint(
insp, "maf_roundtable_seat_runs",
expected_cols={"maf_run_id", "agent_id", "turn_index"},
)
assert old_constraint is None, "old three-column constraint still present after migration"
async with engine.begin() as conn:
await conn.run_sync(_verify)
# Old row back-filled to 'legacy'.
async with sf() as session:
row = (await session.execute(
text("SELECT execution_stage FROM maf_roundtable_seat_runs WHERE id='sr1'")
)).one_or_none()
assert row is not None
assert row[0] == "legacy"
# Re-run the migration — must be idempotent (no error).
async with engine.begin() as conn:
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
async with engine.begin() as conn:
await conn.run_sync(mod._do_migrate_sync, dry_run=True, logger=mod.logger, changes_ref=[0])
# After migration, the store can create rows using the new column, and the
# constraint allows cross_review + finalizer to coexist for the same
# business round (the regression the migration fixes).
store = make_maf_roundtable_store(sf)
await store.seat_runs.create_seat_run(
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="cross_review"
)
await store.seat_runs.create_seat_run(
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="finalizer"
)
rows = await store.seat_runs.get_by_maf_run("r1")
new_mgr_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "mgr"}
assert new_mgr_stages == {"cross_review", "finalizer"}
await engine.dispose()
@@ -22,6 +22,7 @@ Additionally covers:
from __future__ import annotations
import asyncio
import itertools
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
@@ -869,8 +870,9 @@ async def test_audit_event_failure_prevents_external_write(store):
@pytest.mark.asyncio
async def test_deerflow_task_error_marks_run_failed(store):
"""P1: when the DeerFlow agent run raises, the MAF run is marked failed
(not completed), with a sanitized error message."""
"""P3: when a DeerFlow seat turn raises, the seat is soft-failed
(``partial_failure``) and the structured run still completes degraded;
secrets must not appear in persisted seat / run errors."""
svc = _svc(store)
s = await svc.create_session(
user_id="u1",
@@ -890,9 +892,20 @@ async def test_deerflow_task_error_marks_run_failed(store):
await runner.execute(s, run)
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
assert final_run["status"] == "failed"
assert final_run["status"] == "completed"
assert final_run["finished_at"] is not None
assert "sk-secret-12345" not in (final_run.get("error_message_safe") or "")
ckpt = final_run.get("maf_checkpoint") or {}
assert ckpt.get("partial_failure") is True
outcome = (ckpt.get("seat_outcomes") or {}).get("boom-seat") or {}
assert outcome.get("status") in ("failed", "error")
persisted = " ".join(
[
str(final_run.get("error_message_safe") or ""),
str(outcome.get("error") or ""),
str(ckpt),
]
)
assert "sk-secret-12345" not in persisted
# ── 13. Secrets never in events / errors / logs ──────────────────────────
@@ -1200,3 +1213,107 @@ async def test_capability_snapshot_drives_seat_config(store):
caps3 = _agent_capabilities_from_snapshot(None, "seat-1")
assert caps3 == {}
# ── 22. P1-3: Real WorkflowBuilder manager stages via WorkflowRunner ─────────
@pytest.mark.asyncio
async def test_real_workflow_manager_stages_persisted(store):
"""P1-3: drive a REAL MAF WorkflowRunner (real WorkflowBuilder, real SQLite
repository, real adapter + seat-run store) through a structured run with a
manager + 2 seats. Only the runtime client is faked (no real LLM).
Asserts:
- manager has planner, cross_review, finalizer seat-run rows
- cross_review.turn_index == finalizer.turn_index
- cross_review.execution_stage != finalizer.execution_stage
"""
svc = _svc(store)
s = await svc.create_session(
user_id="u1",
body={
"task_text": "分析市场趋势",
"mode": "structured",
"manager_agent_id": "manager-1",
"participants": [
{"agent_id": "manager-1", "role": "manager"},
{"agent_id": "seat-a", "role": "seat"},
{"agent_id": "seat-b", "role": "seat"},
],
},
)
run = await svc.create_run(session_id=s["id"], user_id="u1")
app = _make_app(store)
runner = WorkflowRunner(app, service=svc, user_id="u1")
manager_responses = iter([
"分工:seat-a 负责需求分析,seat-b 负责竞品分析",
json.dumps({
"summary": "审查通过,证据充分",
"evidence_ok": True,
"confirmed_candidates": [],
}),
"最终结论:市场看好,建议推进",
])
seat_responses = {
"seat-a": json.dumps({"summary": "需求旺盛", "findings": ["增长10%"]}),
"seat-b": json.dumps({"summary": "竞品A领先", "findings": ["份额30%"]}),
}
call_log: list[dict] = []
thread_counter = itertools.count()
created_threads: list[str] = []
latest_agent_by_thread: dict[str, str] = {}
async def fake_create_thread(principal):
thread_id = f"thread-{principal.session_id[:8]}-{next(thread_counter)}"
created_threads.append(thread_id)
return thread_id
async def fake_start_agent_run(principal, *, thread_id, agent_id, message, **kwargs):
latest_agent_by_thread[thread_id] = agent_id
call_log.append({"agent_id": agent_id, "thread_id": thread_id, "message": message})
return _FakeRunRecord(run_id=f"df-{agent_id}-{len(call_log)}")
async def fake_read_messages(principal, thread_id):
agent_id = latest_agent_by_thread[thread_id]
if agent_id == "manager-1":
content = next(manager_responses)
else:
content = seat_responses[agent_id]
return [{"type": "ai", "content": content, "usage_metadata": {"input_tokens": 5, "output_tokens": 5}}]
runner.adapter._client.create_thread = fake_create_thread
runner.adapter._client.start_agent_run = fake_start_agent_run
runner.adapter._client.read_thread_messages = fake_read_messages
runner.adapter._client.cancel_run = AsyncMock(return_value=True)
await runner.execute(s, run)
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
assert final_run["status"] == "completed"
seat_runs = await store.seat_runs.get_by_maf_run(run["id"])
mgr_rows = [r for r in seat_runs if r["agent_id"] == "manager-1"]
mgr_stages = {r["execution_stage"] for r in mgr_rows}
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
f"manager stages missing; got {mgr_stages}"
cr_rows = [r for r in mgr_rows if r["execution_stage"] == "cross_review"]
fin_rows = [r for r in mgr_rows if r["execution_stage"] == "finalizer"]
assert len(cr_rows) == 1 and len(fin_rows) == 1
assert cr_rows[0]["turn_index"] == fin_rows[0]["turn_index"]
assert cr_rows[0]["execution_stage"] != fin_rows[0]["execution_stage"]
assert len(created_threads) == 3
assert len(set(created_threads)) == 3
ckpt = final_run["maf_checkpoint"]
assert "最终结论:市场看好" in str(ckpt)
assert any(
r.get("agent_id") == "seat-a"
and (r.get("structured_output") or {}).get("summary") == "需求旺盛"
for r in ckpt["turn_results"]
)
@@ -474,3 +474,43 @@ async def test_identity_from_request_state_not_body(app_and_client):
headers={"X-Test-User": "bob"},
)
assert resp.status_code == 404
# ── Phase 3 resume / recover ──────────────────────────────────────────────
@pytest.mark.asyncio
async def test_resume_and_recover_conflict_when_not_eligible(app_and_client):
"""Resume requires waiting_approval; recover requires running|waiting_approval."""
app, client, _ = app_and_client
app.state.maf_auto_dispatch = False
resp = await client.post(
"/api/maf-roundtable/sessions",
json={"task_text": "t"},
headers={"X-Test-User": "alice"},
)
assert resp.status_code == 200
session_id = resp.json()["id"]
resp = await client.post(
f"/api/maf-roundtable/sessions/{session_id}/runs",
json={},
headers={"X-Test-User": "alice"},
)
assert resp.status_code == 200
run_id = resp.json()["id"]
# Pending runs (auto-dispatch off) are not eligible for resume/recover.
assert resp.json()["status"] == "pending"
resp = await client.post(
f"/api/maf-roundtable/runs/{run_id}/resume",
headers={"X-Test-User": "alice"},
)
assert resp.status_code == 409
resp = await client.post(
f"/api/maf-roundtable/runs/{run_id}/recover",
headers={"X-Test-User": "alice"},
)
assert resp.status_code == 409
@@ -0,0 +1,84 @@
"""Unit tests for seat structured output + one-shot repair."""
from __future__ import annotations
import pytest
from app.maf_roundtable.seat_output import (
build_repair_prompt,
parse_seat_output,
validate_or_repair_seat_output,
)
def test_parse_valid_seat_output():
text = '{"summary":"ok","findings":["f1"],"proposed_actions":["do x"]}'
model, errors = parse_seat_output(text)
assert errors == []
assert model is not None
assert model.summary == "ok"
assert model.findings == ["f1"]
def test_parse_fenced_json():
text = '说明\n```json\n{"summary":"s"}\n```\n'
model, errors = parse_seat_output(text)
assert model is not None
assert errors == []
def test_parse_invalid_missing_summary():
model, errors = parse_seat_output('{"findings":[]}')
assert model is None
assert errors
@pytest.mark.asyncio
async def test_validate_ok_no_repair():
result = await validate_or_repair_seat_output(
raw_content='{"summary":"fine"}',
repair_runner=None,
)
assert result["schema_status"] == "ok"
assert result["repair_attempted"] is False
@pytest.mark.asyncio
async def test_one_shot_repair_success():
calls = {"n": 0}
async def _repair(prompt: str) -> str:
calls["n"] += 1
assert "SeatTurnOutput" in prompt or "校验" in prompt
return '{"summary":"repaired","findings":["x"]}'
result = await validate_or_repair_seat_output(
raw_content="not json at all",
repair_runner=_repair,
)
assert calls["n"] == 1
assert result["schema_status"] == "repaired"
assert result["output"]["summary"] == "repaired"
@pytest.mark.asyncio
async def test_repair_still_invalid_stops():
calls = {"n": 0}
async def _repair(prompt: str) -> str:
calls["n"] += 1
return "still not json"
result = await validate_or_repair_seat_output(
raw_content="garbage",
repair_runner=_repair,
)
assert calls["n"] == 1
assert result["schema_status"] == "schema_invalid"
assert result["repair_attempted"] is True
def test_build_repair_prompt_includes_errors():
prompt = build_repair_prompt("raw", ["summary: Field required"])
assert "summary: Field required" in prompt
assert "raw" in prompt
@@ -0,0 +1,305 @@
"""Phase 3 structured workflow acceptance tests (MAF-03)."""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
import pytest
from maf_test_helpers import make_maf_service
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.maf_roundtable.fact_ledger import executed_action_ids, proposed_action_ids
from app.maf_roundtable.workflow_executors import (
CrossReviewExecutor,
PlannerExecutor,
RoundtableState,
SeatExecutor,
)
from app.maf_roundtable.workflow_factory import create_workflow
from deerflow.persistence.base import Base
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
class _FakeCtx:
def __init__(self) -> None:
self.messages: list[Any] = []
self.outputs: list[Any] = []
async def send_message(self, msg: Any) -> None:
self.messages.append(msg)
async def yield_output(self, msg: Any) -> None:
self.outputs.append(msg)
class _FakeSeatResponse:
def __init__(self, content: str = "", tool_calls: list | None = None, error: str | None = None) -> None:
self.content = content
self.tool_calls = tool_calls or []
self.error = error
self.usage = {"input_tokens": 1, "output_tokens": 1}
def _state(seat_ids: list[str] | None = None) -> RoundtableState:
seats = seat_ids or ["seat-a", "seat-b", "seat-c"]
return RoundtableState(
session_id="s1",
run_id="r1",
task_text="分析市场",
mode="structured",
model_name=None,
reasoning_effort=None,
max_rounds=8,
write_policy="confirm_high_risk",
manager_agent_id=None,
seat_agent_ids=seats,
thread_map={a: f"t-{a}" for a in seats},
capability_snapshot=None,
seat_timeout_seconds=5,
schema_repair_enabled=True,
)
@pytest.mark.asyncio
async def test_planner_deterministic_assignments():
state = _state()
adapter = SimpleNamespace(_emit_critical=AsyncMock())
planner = PlannerExecutor(adapter, state)
ctx = _FakeCtx()
await planner.handle("task", ctx)
assert "planner" in state.completed_nodes
assert state.plan is not None
assert set(state.plan["assignments"]) == {"seat-a", "seat-b", "seat-c"}
assert ctx.messages[-1]["decision"] == "parallel_seats"
@pytest.mark.asyncio
async def test_planner_skips_when_completed():
state = _state()
state.mark_completed("planner")
state.plan = {"source": "deterministic", "assignments": {}}
adapter = SimpleNamespace(run_seat_turn=AsyncMock())
planner = PlannerExecutor(adapter, state)
ctx = _FakeCtx()
await planner.handle("task", ctx)
adapter.run_seat_turn.assert_not_called()
assert ctx.messages[-1].get("replayed") is True
@pytest.mark.asyncio
async def test_seat_partial_failure_does_not_raise():
state = _state(["seat-a", "seat-b"])
state.plan = {
"assignments": {
"seat-a": {"instruction": "do a", "role": "A"},
"seat-b": {"instruction": "do b", "role": "B"},
}
}
state.current_round = 1
async def _run(**kwargs):
if kwargs["agent_id"] == "seat-a":
raise RuntimeError("boom")
return _FakeSeatResponse(content='{"summary":"ok-b","findings":["f"]}')
adapter = SimpleNamespace(
run_seat_turn=AsyncMock(side_effect=_run),
_emit_critical=AsyncMock(),
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
)
seat_a = SeatExecutor(adapter, state, "seat-a")
seat_b = SeatExecutor(adapter, state, "seat-b")
ctx_a, ctx_b = _FakeCtx(), _FakeCtx()
await seat_a.handle({}, ctx_a)
await seat_b.handle({}, ctx_b)
assert state.partial_failure is True
assert state.seat_outcomes["seat-a"]["status"] in ("failed", "error")
assert state.seat_outcomes["seat-b"]["status"] == "completed"
assert any(r.get("agent_id") == "seat-b" and r.get("content") for r in state.turn_results)
assert "seat_seat-a" in state.completed_nodes
assert "seat_seat-b" in state.completed_nodes
@pytest.mark.asyncio
async def test_seat_schema_one_repair_then_invalid():
state = _state(["seat-a"])
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
state.current_round = 1
calls = {"n": 0}
async def _run(**kwargs):
calls["n"] += 1
if calls["n"] == 1:
return _FakeSeatResponse(content="not-json")
return _FakeSeatResponse(content="still-bad")
adapter = SimpleNamespace(
run_seat_turn=AsyncMock(side_effect=_run),
_emit_critical=AsyncMock(),
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
)
seat = SeatExecutor(adapter, state, "seat-a")
await seat.handle({}, _FakeCtx())
# 1 expert + 1 repair
assert calls["n"] == 2
row = state.turn_results[-1]
assert row["schema_status"] == "schema_invalid"
@pytest.mark.asyncio
async def test_proposed_not_executed_in_ledger():
state = _state(["seat-a"])
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
state.current_round = 1
content = (
'{"summary":"s","findings":["f"],'
'"proposed_actions":[{"action_id":"p1","text":"write"}],'
'"executed_actions":[{"action_id":"forge","text":"lie"}]}'
)
adapter = SimpleNamespace(
run_seat_turn=AsyncMock(return_value=_FakeSeatResponse(content=content)),
_emit_critical=AsyncMock(),
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
)
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
assert "p1" in proposed_action_ids(state.fact_ledger)
assert "forge" not in executed_action_ids(state.fact_ledger)
@pytest.mark.asyncio
async def test_cross_review_barrier_marks_missing():
state = _state(["seat-a", "seat-b"])
state.seat_outcomes = {"seat-a": {"status": "completed"}}
state.current_round = 1
adapter = SimpleNamespace(run_seat_turn=AsyncMock(), _emit_critical=AsyncMock())
review = CrossReviewExecutor(adapter, state)
await review.handle({}, _FakeCtx())
assert state.seat_outcomes["seat-b"]["status"] == "missing"
assert state.partial_failure is True
assert "cross_review" in state.completed_nodes
def test_structured_factory_requires_planner():
with pytest.raises(ValueError):
create_workflow(
mode="structured",
seat_executors=[],
approval_gate=object(),
finalizer=object(),
waiting_terminal=object(),
seat_agent_ids=[],
)
@pytest.mark.asyncio
async def test_recover_skips_completed_seats(tmp_path, monkeypatch):
"""completed_nodes must not re-invoke run_seat_turn on recover."""
pytest.importorskip("agent_framework")
from app.maf_roundtable.workflow_runner import WorkflowRunner
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'p3r.db'}")
async with engine.begin() as conn:
import deerflow.persistence.maf_roundtable.model # noqa: F401
await conn.run_sync(Base.metadata.create_all)
sf = async_sessionmaker(engine, expire_on_commit=False)
store = make_maf_roundtable_store(sf)
svc = make_maf_service(store)
session = await svc.create_session(
user_id="u1",
body={
"task_text": "t",
"mode": "structured",
"participants": [
{"agent_id": "seat-a", "role": "seat"},
{"agent_id": "seat-b", "role": "seat"},
],
},
)
run = await svc.create_run(session_id=session["id"], user_id="u1")
# Pretend planner + seat-a already finished; seat-b not yet.
state = RoundtableState(
session_id=session["id"],
run_id=run["id"],
task_text="t",
mode="structured",
model_name=None,
reasoning_effort=None,
max_rounds=8,
write_policy="confirm_high_risk",
manager_agent_id=None,
seat_agent_ids=["seat-a", "seat-b"],
thread_map={"seat-a": "ta", "seat-b": "tb"},
capability_snapshot=session.get("capability_snapshot"),
seat_timeout_seconds=30,
schema_repair_enabled=False,
)
state.plan = {
"source": "deterministic",
"assignments": {
"seat-a": {"instruction": "a", "role": "A"},
"seat-b": {"instruction": "b", "role": "B"},
},
}
state.current_round = 1
state.stage = "experts"
state.mark_completed("planner")
state.mark_completed("seat_seat-a")
state.seat_outcomes["seat-a"] = {"status": "completed"}
state.turn_results.append({
"agent_id": "seat-a",
"turn_index": 1,
"content": '{"summary":"a"}',
"tool_calls": [],
"error": None,
})
await store.runs.update_run(run["id"], {"status": "running", "maf_checkpoint": state.to_dict()})
calls: list[str] = []
async def _run_seat(**kwargs):
aid = kwargs["agent_id"]
calls.append(aid)
return _FakeSeatResponse(content=f'{{"summary":"{aid}"}}')
app = SimpleNamespace(state=SimpleNamespace(maf_roundtable_service=svc, checkpointer=None))
runner = WorkflowRunner(app, service=svc, user_id="u1")
monkeypatch.setattr(runner._adapter, "run_seat_turn", AsyncMock(side_effect=_run_seat))
monkeypatch.setattr(runner._adapter, "ensure_threads", AsyncMock(return_value=state.thread_map))
monkeypatch.setattr(runner._adapter, "_emit_critical", AsyncMock())
# Avoid needing a real SDK stream — drive recover path by patching _run_workflow
# to invoke remaining executors with skip semantics.
async def _fake_run_workflow(workflow, st):
# Simulate graph: planner (skip) → seats → cross_review → approval → finalizer
from app.maf_roundtable.workflow_executors import (
ApprovalGateExecutor,
CrossReviewExecutor,
FinalizerExecutor,
PlannerExecutor,
SeatExecutor,
)
ctx = _FakeCtx()
await PlannerExecutor(runner._adapter, st).handle("t", ctx)
for aid in st.seat_agent_ids:
await SeatExecutor(runner._adapter, st, aid).handle({}, _FakeCtx())
await CrossReviewExecutor(runner._adapter, st).handle({}, _FakeCtx())
await ApprovalGateExecutor(runner._adapter, st, svc).handle({}, _FakeCtx())
await FinalizerExecutor(runner._adapter, st).handle({}, _FakeCtx())
st.status = "completed"
monkeypatch.setattr(runner, "_run_workflow", _fake_run_workflow)
try:
await runner.recover(run["id"])
# seat-a was completed — must not be called; seat-b should run once.
assert "seat-a" not in calls
assert calls.count("seat-b") == 1
finally:
await engine.dispose()