feat(maf): complete resilient roundtable workflow
This commit is contained in:
1 parent
bd4131b4e2
commit
21c4c898e7
23 files changed
+4810
-312
No files matched your search
@@ -7,7 +7,8 @@ Phase 1 ships:
|
||||
- Session CRUD: ``POST /sessions``, ``GET /sessions``, ``GET /sessions/{id}``,
|
||||
``PUT /sessions/{id}``, ``DELETE /sessions/{id}``.
|
||||
- Run lifecycle: ``POST /sessions/{id}/runs``, ``GET /runs/{id}``,
|
||||
``GET /sessions/{id}/runs``, ``POST /runs/{id}/cancel``.
|
||||
``GET /sessions/{id}/runs``, ``POST /runs/{id}/cancel``,
|
||||
``POST /runs/{id}/resume``, ``POST /runs/{id}/recover``.
|
||||
- Event stream: ``GET /runs/{id}/events?after_seq=N`` (断线补拉) — **read-only**;
|
||||
there is no public ``POST /runs/{id}/events``. Real workflow events are
|
||||
written only by the backend executor via ``service.append_event``.
|
||||
@@ -32,7 +33,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi import APIRouter, HTTPException, Query, Request, status
|
||||
|
||||
from app.maf_roundtable import (
|
||||
ActionListResponse,
|
||||
@@ -291,6 +292,76 @@ async def cancel_run(request: Request, run_id: str) -> RunResponse:
|
||||
return RunResponse(**final_run)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/resume", response_model=RunResponse)
|
||||
async def resume_run(request: Request, run_id: str) -> RunResponse:
|
||||
"""Resume after approvals are decided (owner-only).
|
||||
|
||||
If the run is not ``waiting_approval`` or still has pending approvals,
|
||||
returns 409.
|
||||
"""
|
||||
svc = _require_service(request)
|
||||
user_id, is_admin = _user_ctx(request)
|
||||
try:
|
||||
run = await svc.get_run(run_id=run_id, user_id=user_id, is_admin=False)
|
||||
except (MafDisabledError, MafForbiddenError, MafNotFoundError) as exc:
|
||||
raise _map_service_error(exc) from exc
|
||||
|
||||
if run.get("status") != "waiting_approval":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"run {run_id} is not waiting_approval (status={run.get('status')})",
|
||||
)
|
||||
|
||||
from app.maf_roundtable.dispatcher import get_dispatcher
|
||||
|
||||
dispatcher = get_dispatcher(request.app)
|
||||
try:
|
||||
updated = await dispatcher.resume(run_id, user_id=user_id or "default")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"resume failed: {type(exc).__name__}",
|
||||
) from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="run not found")
|
||||
return RunResponse(**updated)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/recover", response_model=RunResponse)
|
||||
async def recover_run(request: Request, run_id: str) -> RunResponse:
|
||||
"""Graph-level recover after process restart (owner-only).
|
||||
|
||||
Re-enters the workflow from checkpoint; completed_nodes are skipped.
|
||||
Does not silently scan all runs at startup.
|
||||
"""
|
||||
svc = _require_service(request)
|
||||
user_id, is_admin = _user_ctx(request)
|
||||
try:
|
||||
run = await svc.get_run(run_id=run_id, user_id=user_id, is_admin=False)
|
||||
except (MafDisabledError, MafForbiddenError, MafNotFoundError) as exc:
|
||||
raise _map_service_error(exc) from exc
|
||||
|
||||
if run.get("status") not in ("running", "waiting_approval"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"run {run_id} cannot be recovered (status={run.get('status')})",
|
||||
)
|
||||
|
||||
from app.maf_roundtable.dispatcher import get_dispatcher
|
||||
|
||||
dispatcher = get_dispatcher(request.app)
|
||||
try:
|
||||
updated = await dispatcher.recover(run_id, user_id=user_id or "default")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"recover failed: {type(exc).__name__}",
|
||||
) from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="run not found")
|
||||
return RunResponse(**updated)
|
||||
|
||||
|
||||
# ── events ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -127,11 +127,35 @@ def _format_prior_turns(turns: list[dict[str, Any]], max_per_turn: int) -> str:
|
||||
def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
parts: list[str] = []
|
||||
|
||||
candidates = ledger.get("candidate_facts")
|
||||
if candidates and isinstance(candidates, list):
|
||||
# Phase-3: candidate facts are UNVERIFIED seat claims — label them
|
||||
# explicitly so downstream seats / the finalizer never treat them as
|
||||
# established conclusions.
|
||||
parts.append("未经核验的席位主张(不能作为结论):")
|
||||
for c in candidates[:20]:
|
||||
if isinstance(c, dict):
|
||||
parts.append(f" - {c.get('text', c)}")
|
||||
else:
|
||||
parts.append(f" - {c}")
|
||||
|
||||
facts = ledger.get("confirmed_facts")
|
||||
if facts and isinstance(facts, list):
|
||||
parts.append("已确认事实:")
|
||||
parts.append("已确认事实(经交叉审查核验):")
|
||||
for f in facts[:20]:
|
||||
parts.append(f" - {f}")
|
||||
if isinstance(f, dict):
|
||||
parts.append(f" - {f.get('text', f)}")
|
||||
else:
|
||||
parts.append(f" - {f}")
|
||||
|
||||
evidence = ledger.get("evidence")
|
||||
if evidence and isinstance(evidence, list):
|
||||
parts.append("证据引用:")
|
||||
for e in evidence[:10]:
|
||||
if isinstance(e, dict):
|
||||
parts.append(f" - {e.get('ref') or e.get('text') or e}")
|
||||
else:
|
||||
parts.append(f" - {e}")
|
||||
|
||||
tool_results = ledger.get("tool_results")
|
||||
if tool_results and isinstance(tool_results, list):
|
||||
@@ -142,6 +166,21 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
else:
|
||||
parts.append(f" - {tr}")
|
||||
|
||||
proposed = ledger.get("proposed_actions")
|
||||
if proposed and isinstance(proposed, list):
|
||||
parts.append("建议动作(未执行):")
|
||||
for p in proposed[:8]:
|
||||
parts.append(f" - {p}")
|
||||
|
||||
executed = ledger.get("executed_actions")
|
||||
if executed and isinstance(executed, list):
|
||||
parts.append("已执行外部写入:")
|
||||
for e in executed[:8]:
|
||||
if isinstance(e, dict):
|
||||
parts.append(f" - {e.get('tool_name', '?')} ({e.get('action_id', '')})")
|
||||
else:
|
||||
parts.append(f" - {e}")
|
||||
|
||||
conflicts = ledger.get("conflicts")
|
||||
if conflicts and isinstance(conflicts, list):
|
||||
parts.append("未解决冲突:")
|
||||
@@ -154,6 +193,12 @@ def _format_fact_ledger(ledger: dict[str, Any], max_chars: int) -> str:
|
||||
for p in pending[:5]:
|
||||
parts.append(f" - {p}")
|
||||
|
||||
failed = ledger.get("failed_or_unknown_actions")
|
||||
if failed and isinstance(failed, list):
|
||||
parts.append("失败或状态不确定的动作:")
|
||||
for f in failed[:5]:
|
||||
parts.append(f" - {f}")
|
||||
|
||||
result = "\n".join(parts)
|
||||
if len(result) > max_chars:
|
||||
result = result[:max_chars]
|
||||
|
||||
@@ -13,7 +13,8 @@ Architecture:
|
||||
- Cancellation propagates from MAF run → DeerFlow run via RunManager.cancel.
|
||||
- All seat runs persisted to ``maf_roundtable_seat_runs`` (survives restart).
|
||||
- Critical audit events (tool.*, skill.*, approval.*, artifact.*) MUST persist;
|
||||
failure stops external writes and marks the run as failed.
|
||||
audit failure stops external writes; seat turn failures are soft-failed
|
||||
into ``seat_outcomes`` / ``partial_failure`` (structured mode).
|
||||
|
||||
Boundary: lives in ``app.maf_roundtable`` (may import ``deerflow.*`` and
|
||||
``app.gateway.*``); never imported by ``deerflow.*``.
|
||||
@@ -132,6 +133,7 @@ class DeerFlowAgentAdapter:
|
||||
turn_index: int,
|
||||
message: str,
|
||||
thread_id: str,
|
||||
execution_stage: str = "expert",
|
||||
model_name: str | None = None,
|
||||
reasoning_effort: str | None = None,
|
||||
thinking_enabled: bool = True,
|
||||
@@ -147,6 +149,7 @@ class DeerFlowAgentAdapter:
|
||||
capability_snapshot: dict[str, Any] | None = None,
|
||||
write_policy: str = "confirm_high_risk",
|
||||
approved_tool: str | None = None,
|
||||
wait_timeout_seconds: float | None = None,
|
||||
) -> SeatResponse:
|
||||
"""Execute one seat turn through the DeerFlow Agent Runtime."""
|
||||
t0 = time.perf_counter()
|
||||
@@ -166,10 +169,56 @@ class DeerFlowAgentAdapter:
|
||||
|
||||
principal = ExecutionPrincipal(user_id=self._user_id, session_id=session_id, run_id=run_id)
|
||||
|
||||
seat_run = await self._service.store.seat_runs.create_seat_run(
|
||||
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index, deerflow_thread_id=thread_id
|
||||
# Phase-3 P0-1: atomic "occupy or read". Only the caller that wins the
|
||||
# insert (created=True) may start a DeerFlow run. An existing row is
|
||||
# handled by status so we NEVER silently start a second DeerFlow
|
||||
# execution for the same four-tuple (run, agent, turn, stage).
|
||||
seat_run, created = await self._service.store.seat_runs.get_or_create_seat_run(
|
||||
maf_run_id=run_id, agent_id=agent_id, turn_index=turn_index,
|
||||
execution_stage=execution_stage, deerflow_thread_id=thread_id,
|
||||
)
|
||||
seat_run_id = seat_run["id"]
|
||||
prior_status = seat_run.get("status")
|
||||
|
||||
if not created:
|
||||
if prior_status == "completed":
|
||||
return self._seat_response_from_snapshot(
|
||||
seat_run=seat_run,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
if prior_status in ("pending", "running", "cancelling"):
|
||||
effective_timeout = wait_timeout_seconds if wait_timeout_seconds is not None else 600.0
|
||||
seat_run = await self._service.store.seat_runs.wait_for_terminal_seat_run(
|
||||
maf_run_id=run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
timeout_seconds=effective_timeout,
|
||||
)
|
||||
if seat_run["status"] == "completed":
|
||||
return self._seat_response_from_snapshot(
|
||||
seat_run=seat_run,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error=f"stage '{execution_stage}' ended as {seat_run['status']}",
|
||||
metadata={"seat_run_id": seat_run["id"], "terminal": seat_run["status"]},
|
||||
latency_ms=int((time.perf_counter() - t0) * 1000),
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error=f"stage '{execution_stage}' previously terminal ({prior_status}); "
|
||||
f"manual recovery required",
|
||||
metadata={"terminal": prior_status, "seat_run_id": seat_run_id},
|
||||
latency_ms=int((time.perf_counter() - t0) * 1000),
|
||||
)
|
||||
|
||||
try:
|
||||
response = await self._execute_run(
|
||||
@@ -210,7 +259,16 @@ class DeerFlowAgentAdapter:
|
||||
latency_ms=latency,
|
||||
)
|
||||
|
||||
await self._service.store.seat_runs.mark_completed(seat_run_id)
|
||||
raw_snapshot = {
|
||||
"content": response.content,
|
||||
"tool_calls": response.tool_calls,
|
||||
"usage": response.usage,
|
||||
"metadata": response.metadata,
|
||||
}
|
||||
snapshot = sanitize(raw_snapshot)
|
||||
if not isinstance(snapshot, dict):
|
||||
raise RuntimeError("seat result snapshot sanitization produced invalid payload")
|
||||
await self._service.store.seat_runs.mark_completed(seat_run_id, result_snapshot=snapshot)
|
||||
response.latency_ms = int((time.perf_counter() - t0) * 1000)
|
||||
await self._emit_critical(run_id, make_turn_completed(
|
||||
agent_id, turn_index,
|
||||
@@ -463,6 +521,34 @@ class DeerFlowAgentAdapter:
|
||||
|
||||
# ── internal helpers ────────────────────────────────────────────────────
|
||||
|
||||
def _seat_response_from_snapshot(
|
||||
self,
|
||||
*,
|
||||
seat_run: dict[str, Any],
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
) -> SeatResponse:
|
||||
snapshot = seat_run.get("result_snapshot")
|
||||
if not snapshot or not isinstance(snapshot, dict):
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content="",
|
||||
error="completed seat run has no replayable snapshot; manual recovery required",
|
||||
metadata={"seat_run_id": seat_run.get("id"), "terminal": seat_run.get("status")},
|
||||
latency_ms=0,
|
||||
)
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content=str(snapshot.get("content", "")),
|
||||
tool_calls=list(snapshot.get("tool_calls") or []),
|
||||
usage=snapshot.get("usage"),
|
||||
metadata=dict(snapshot.get("metadata") or {}),
|
||||
error=None,
|
||||
latency_ms=0,
|
||||
)
|
||||
|
||||
def _latest_ai_text(self, messages: list[dict[str, Any]]) -> str:
|
||||
for m in reversed(messages):
|
||||
if not isinstance(m, dict):
|
||||
|
||||
@@ -96,6 +96,36 @@ class MafRunDispatcher:
|
||||
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
|
||||
return await runner.resume(run_id)
|
||||
|
||||
async def recover(self, run_id: str, *, user_id: str) -> dict[str, Any] | None:
|
||||
"""Re-enter the workflow from checkpoint after a process restart.
|
||||
|
||||
Unlike ``resume`` (approval path), this continues orchestration for
|
||||
``running`` runs and skips ``completed_nodes``.
|
||||
"""
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
service = getattr(self._app.state, "maf_roundtable_service", None)
|
||||
if service is None:
|
||||
raise MafSdkNotAvailableError("MAF roundtable service not available")
|
||||
|
||||
# Avoid overlapping with an in-process execute task.
|
||||
existing = self._tasks.get(run_id)
|
||||
if existing is not None and not existing.done():
|
||||
logger.info("recover(%s): background task still running — skip", run_id)
|
||||
return await service.store.runs.get_run(run_id)
|
||||
|
||||
runner = WorkflowRunner(self._app, service=service, user_id=user_id)
|
||||
task = asyncio.create_task(self._recover(runner, run_id))
|
||||
self._tasks[run_id] = task
|
||||
task.add_done_callback(lambda t, rid=run_id: self._tasks.pop(rid, None))
|
||||
return await service.store.runs.get_run(run_id)
|
||||
|
||||
async def _recover(self, runner: Any, run_id: str) -> None:
|
||||
try:
|
||||
await runner.recover(run_id)
|
||||
except Exception:
|
||||
logger.exception("MAF recover failed for run %s", run_id)
|
||||
|
||||
async def cancel(self, run_id: str) -> dict[str, Any]:
|
||||
"""Cancel a running workflow AND cascade to DeerFlow sub-runs.
|
||||
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Shared fact ledger for MAF roundtable runs (Phase 3).
|
||||
|
||||
The ledger is the authoritative record of confirmed facts, tool outcomes, and
|
||||
external-action status. Chat text must never be treated as proof that a write
|
||||
succeeded — hosts synthesize conclusions from this structure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
_TOOL_STATUSES = frozenset({"completed", "waiting_approval", "failed", "unknown"})
|
||||
_ACTION_STATUSES = frozenset({"pending", "executed", "failed", "unknown", "rejected"})
|
||||
|
||||
|
||||
def empty_ledger() -> dict[str, Any]:
|
||||
"""Return a fresh ledger with all Phase-3 partitions.
|
||||
|
||||
Phase-3 separation of evidence trust levels:
|
||||
- ``candidate_facts`` — unverified seat claims; the model said it, that
|
||||
does not make it true. Each carries a server-generated ``candidate_id``.
|
||||
- ``confirmed_facts`` — only facts a cross-review confirmed (with evidence
|
||||
and a confirmer) ever land here.
|
||||
- ``tool_results`` — outcomes of tools that actually ran.
|
||||
- ``pending_actions`` / ``executed_actions`` / ``failed_or_unknown_actions``
|
||||
— external writes partitioned by lifecycle state.
|
||||
"""
|
||||
return {
|
||||
"candidate_facts": [],
|
||||
"confirmed_facts": [],
|
||||
"evidence": [],
|
||||
"tool_results": [],
|
||||
"executed_actions": [],
|
||||
"pending_actions": [],
|
||||
"failed_or_unknown_actions": [],
|
||||
"conflicts": [],
|
||||
# Legacy alias kept for old checkpoints / prompts.
|
||||
"proposed_actions": [],
|
||||
}
|
||||
|
||||
|
||||
def normalize_ledger(raw: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Coerce a possibly-partial / legacy ledger into the full shape (in-place when possible)."""
|
||||
if not isinstance(raw, dict):
|
||||
return empty_ledger()
|
||||
base = empty_ledger()
|
||||
for key in base:
|
||||
if key not in raw or not isinstance(raw.get(key), list):
|
||||
raw[key] = []
|
||||
return raw
|
||||
|
||||
|
||||
def candidate_id_map(ledger: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
"""Return ``{candidate_id: candidate_entry}`` for O(1) confirmation lookup."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(c["candidate_id"]): c
|
||||
for c in ledger["candidate_facts"]
|
||||
if isinstance(c, dict) and c.get("candidate_id")
|
||||
}
|
||||
|
||||
|
||||
def known_evidence_refs(ledger: dict[str, Any]) -> set[str]:
|
||||
"""The set of evidence references the ledger actually has backing for.
|
||||
|
||||
A cross-review may only confirm a candidate using evidence that the ledger
|
||||
already recorded (either in ``evidence`` or as a successful
|
||||
``tool_results`` entry). This is the trust anchor that stops a weak model
|
||||
from inventing evidence references to promote a seat claim.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
refs: set[str] = set()
|
||||
for e in ledger["evidence"]:
|
||||
if isinstance(e, dict):
|
||||
r = e.get("ref") or e.get("text")
|
||||
if r:
|
||||
refs.add(str(r))
|
||||
elif e:
|
||||
refs.add(str(e))
|
||||
for tr in ledger["tool_results"]:
|
||||
# Only successful tool runs count as evidence (a failed tool proves
|
||||
# nothing). Each tool result is addressable as tool:<name>:<turn>.
|
||||
if isinstance(tr, dict) and tr.get("status") == "completed":
|
||||
name = tr.get("tool_name", "")
|
||||
turn = tr.get("turn_index")
|
||||
if name:
|
||||
refs.add(f"tool:{name}:{turn}" if turn is not None else f"tool:{name}")
|
||||
return refs
|
||||
|
||||
|
||||
def merge_seat_output(
|
||||
ledger: dict[str, Any],
|
||||
seat_output: dict[str, Any],
|
||||
*,
|
||||
agent_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge a validated seat structured output into the ledger.
|
||||
|
||||
Phase-3: seat ``findings`` are written to ``candidate_facts`` (unverified
|
||||
claims), NOT to ``confirmed_facts``. Promotion to ``confirmed_facts``
|
||||
happens only via :func:`confirm_candidate_facts` after cross-review.
|
||||
Proposed actions stay in ``proposed_actions`` — they are never promoted to
|
||||
``executed_actions`` here.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
for finding in seat_output.get("findings") or []:
|
||||
text = str(finding).strip()
|
||||
if text:
|
||||
ledger["candidate_facts"].append({
|
||||
"candidate_id": uuid4().hex,
|
||||
"text": text,
|
||||
"agent_id": agent_id,
|
||||
"source": "seat_finding",
|
||||
"confirmed": False,
|
||||
})
|
||||
for ref in seat_output.get("evidence_refs") or []:
|
||||
if isinstance(ref, dict):
|
||||
entry = dict(ref)
|
||||
entry.setdefault("agent_id", agent_id)
|
||||
ledger["evidence"].append(entry)
|
||||
elif ref:
|
||||
ledger["evidence"].append({"ref": str(ref), "agent_id": agent_id})
|
||||
for conflict in seat_output.get("conflicts") or []:
|
||||
if conflict:
|
||||
ledger["conflicts"].append({"text": str(conflict), "agent_id": agent_id})
|
||||
for action in seat_output.get("proposed_actions") or []:
|
||||
if isinstance(action, dict):
|
||||
entry = dict(action)
|
||||
entry.setdefault("agent_id", agent_id)
|
||||
entry.setdefault("status", "proposed")
|
||||
ledger["proposed_actions"].append(entry)
|
||||
elif action:
|
||||
ledger["proposed_actions"].append({"text": str(action), "agent_id": agent_id, "status": "proposed"})
|
||||
# Executed actions from model text are NOT trusted — only ActionCoordinator
|
||||
# / resume may write executed_actions. Ignore seat_output.executed_actions.
|
||||
return ledger
|
||||
|
||||
|
||||
def confirm_candidates(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
confirmed_candidates: list[dict[str, Any]],
|
||||
confirmed_by: str,
|
||||
rejections: list[dict[str, Any]] | None = None,
|
||||
accepted: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Promote candidate facts to ``confirmed_facts`` after cross-review.
|
||||
|
||||
Phase-3 P1-1 — server-side evidence binding. A candidate is promoted ONLY
|
||||
when **all** of the following hold:
|
||||
|
||||
1. ``candidate_id`` exists in ``candidate_facts`` (the model cannot invent a
|
||||
confirmation for a fact a seat never asserted);
|
||||
2. the candidate carries **at least one** evidence reference;
|
||||
3. every evidence reference resolves to an entry the ledger actually
|
||||
recorded — either in ``evidence`` or as a **successful** ``tool_results``
|
||||
entry. An evidence ref the model invented (not in the ledger) is a
|
||||
hallucination and blocks promotion.
|
||||
|
||||
Candidates that fail any check are left as candidates (``confirmed=False``)
|
||||
and a rejection reason is appended to ``rejections`` so the cross-review
|
||||
summary can explain why a claim was not trusted. When supplied, ``accepted``
|
||||
receives only the server-validated confirmations, so callers never need to
|
||||
infer success from the model's requested confirmation list. Idempotent: a
|
||||
candidate already present in ``confirmed_facts`` is not re-confirmed.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
if rejections is None:
|
||||
rejections = []
|
||||
by_id = candidate_id_map(ledger)
|
||||
valid_refs = known_evidence_refs(ledger)
|
||||
existing_ids = {
|
||||
str(f.get("candidate_id"))
|
||||
for f in ledger["confirmed_facts"]
|
||||
if isinstance(f, dict) and f.get("candidate_id")
|
||||
}
|
||||
for item in confirmed_candidates or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
cid = str(item.get("candidate_id", "")).strip()
|
||||
refs = list(item.get("evidence_refs") or [])
|
||||
if not cid:
|
||||
rejections.append({"reason": "missing_candidate_id"})
|
||||
continue
|
||||
cand = by_id.get(cid)
|
||||
if cand is None:
|
||||
rejections.append({"candidate_id": cid, "reason": "unknown_candidate_id"})
|
||||
continue
|
||||
if cid in existing_ids:
|
||||
# Already confirmed — idempotent no-op.
|
||||
continue
|
||||
if not refs:
|
||||
rejections.append({"candidate_id": cid, "reason": "no_evidence"})
|
||||
continue
|
||||
bad_refs = [r for r in refs if str(r) not in valid_refs]
|
||||
if bad_refs:
|
||||
rejections.append({
|
||||
"candidate_id": cid,
|
||||
"reason": "evidence_not_in_ledger",
|
||||
"evidence_refs": bad_refs,
|
||||
})
|
||||
continue
|
||||
cand["confirmed"] = True
|
||||
ledger["confirmed_facts"].append({
|
||||
"candidate_id": cid,
|
||||
"text": cand.get("text", ""),
|
||||
"agent_id": cand.get("agent_id"),
|
||||
"source": "cross_review_confirmed",
|
||||
"confirmed_by": confirmed_by,
|
||||
"evidence_refs": list(refs),
|
||||
})
|
||||
if accepted is not None:
|
||||
accepted.append({
|
||||
"candidate_id": cid,
|
||||
"evidence_refs": list(refs),
|
||||
})
|
||||
existing_ids.add(cid)
|
||||
return ledger
|
||||
|
||||
|
||||
def confirm_candidate_facts(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
confirmed_candidate_ids: list[str],
|
||||
confirmed_by: str,
|
||||
evidence_refs: list[str] | None = None,
|
||||
accepted: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Back-compat wrapper: confirm a flat id list against one shared evidence set.
|
||||
|
||||
Phase-3 P1-1 tightened confirmation so each candidate must carry at least
|
||||
one evidence reference that the ledger has backing for. The structured
|
||||
path (:func:`confirm_candidates`) is the primary API; this wrapper exists
|
||||
for callers that still pass a flat ``confirmed_candidate_ids`` list — it
|
||||
promotes a candidate ONLY when ``evidence_refs`` is non-empty AND every
|
||||
ref resolves to ledger evidence / successful tool results.
|
||||
"""
|
||||
shared_refs = list(evidence_refs or [])
|
||||
candidates = [
|
||||
{"candidate_id": cid, "evidence_refs": shared_refs}
|
||||
for cid in confirmed_candidate_ids or []
|
||||
]
|
||||
return confirm_candidates(
|
||||
ledger,
|
||||
confirmed_candidates=candidates,
|
||||
confirmed_by=confirmed_by,
|
||||
accepted=accepted,
|
||||
)
|
||||
|
||||
|
||||
def record_tool_result(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
tool_name: str,
|
||||
status: str,
|
||||
agent_id: str,
|
||||
turn_index: int | None = None,
|
||||
detail: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a tool result. ``status`` must be one of the allowed tool statuses."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
st = status if status in _TOOL_STATUSES else "unknown"
|
||||
entry: dict[str, Any] = {
|
||||
"tool_name": tool_name,
|
||||
"status": st,
|
||||
"agent_id": agent_id,
|
||||
}
|
||||
if turn_index is not None:
|
||||
entry["turn_index"] = turn_index
|
||||
if detail:
|
||||
entry["detail"] = detail
|
||||
ledger["tool_results"].append(entry)
|
||||
return ledger
|
||||
|
||||
|
||||
def record_action_transition(
|
||||
ledger: dict[str, Any],
|
||||
*,
|
||||
action_id: str,
|
||||
tool_name: str,
|
||||
status: str,
|
||||
agent_id: str | None = None,
|
||||
external_object_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Move / record an external action under the correct ledger partition.
|
||||
|
||||
Rules:
|
||||
- ``proposed`` never becomes ``executed`` via this helper unless status is
|
||||
explicitly ``executed`` (called only after ToolExecutor success).
|
||||
- ``pending`` entries are removed when transitioning to a terminal status.
|
||||
"""
|
||||
ledger = normalize_ledger(ledger)
|
||||
st = status if status in _ACTION_STATUSES else "unknown"
|
||||
entry: dict[str, Any] = {
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"status": st,
|
||||
}
|
||||
if agent_id:
|
||||
entry["agent_id"] = agent_id
|
||||
if external_object_id:
|
||||
entry["external_object_id"] = external_object_id
|
||||
|
||||
# Drop matching pending rows when leaving pending.
|
||||
if st != "pending":
|
||||
ledger["pending_actions"] = [
|
||||
p
|
||||
for p in ledger["pending_actions"]
|
||||
if not (isinstance(p, dict) and p.get("action_id") == action_id)
|
||||
]
|
||||
|
||||
if st == "pending":
|
||||
# Replace existing pending with same action_id.
|
||||
ledger["pending_actions"] = [
|
||||
p
|
||||
for p in ledger["pending_actions"]
|
||||
if not (isinstance(p, dict) and p.get("action_id") == action_id)
|
||||
]
|
||||
ledger["pending_actions"].append(entry)
|
||||
elif st == "executed":
|
||||
ledger["executed_actions"] = [
|
||||
e
|
||||
for e in ledger["executed_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
ledger["executed_actions"].append(entry)
|
||||
# Strip from failed list if previously unknown/failed.
|
||||
ledger["failed_or_unknown_actions"] = [
|
||||
e
|
||||
for e in ledger["failed_or_unknown_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
elif st in ("failed", "unknown", "rejected"):
|
||||
ledger["failed_or_unknown_actions"] = [
|
||||
e
|
||||
for e in ledger["failed_or_unknown_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
ledger["failed_or_unknown_actions"].append(entry)
|
||||
# Never keep executed copy of a failed/unknown action.
|
||||
ledger["executed_actions"] = [
|
||||
e
|
||||
for e in ledger["executed_actions"]
|
||||
if not (isinstance(e, dict) and e.get("action_id") == action_id)
|
||||
]
|
||||
return ledger
|
||||
|
||||
|
||||
def assert_ledger_invariants(ledger: dict[str, Any]) -> None:
|
||||
"""Raise AssertionError if the ledger violates evidence/action constraints."""
|
||||
ledger = normalize_ledger(ledger)
|
||||
executed_ids = {
|
||||
e.get("action_id")
|
||||
for e in ledger["executed_actions"]
|
||||
if isinstance(e, dict) and e.get("action_id")
|
||||
}
|
||||
for prop in ledger["proposed_actions"]:
|
||||
if not isinstance(prop, dict):
|
||||
continue
|
||||
# A proposed-only entry must not also appear as executed with status proposed.
|
||||
if prop.get("status") == "proposed" and prop.get("action_id") in executed_ids:
|
||||
# Same action_id may be proposed then later executed — that is OK if
|
||||
# the executed entry has status executed. Only flag if proposed row
|
||||
# itself claims executed.
|
||||
if prop.get("status") == "executed":
|
||||
raise AssertionError("proposed_actions entry has status=executed")
|
||||
for e in ledger["executed_actions"]:
|
||||
if isinstance(e, dict) and e.get("status") not in (None, "executed"):
|
||||
raise AssertionError(f"executed_actions entry has status={e.get('status')!r}")
|
||||
for tr in ledger["tool_results"]:
|
||||
if isinstance(tr, dict) and tr.get("status") not in _TOOL_STATUSES:
|
||||
raise AssertionError(f"invalid tool_results status={tr.get('status')!r}")
|
||||
|
||||
|
||||
def proposed_action_ids(ledger: dict[str, Any]) -> set[str]:
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(p["action_id"])
|
||||
for p in ledger["proposed_actions"]
|
||||
if isinstance(p, dict) and p.get("action_id")
|
||||
}
|
||||
|
||||
|
||||
def executed_action_ids(ledger: dict[str, Any]) -> set[str]:
|
||||
ledger = normalize_ledger(ledger)
|
||||
return {
|
||||
str(e["action_id"])
|
||||
for e in ledger["executed_actions"]
|
||||
if isinstance(e, dict) and e.get("action_id")
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Seat turn structured-output schema + one-shot format repair (Phase 3)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
RepairRunner = Callable[[str], Awaitable[str]]
|
||||
|
||||
|
||||
class SeatTurnOutput(BaseModel):
|
||||
"""Validated structured output produced by an expert seat turn."""
|
||||
|
||||
summary: str = Field(..., min_length=1, description="Short natural-language summary")
|
||||
findings: list[str] = Field(default_factory=list)
|
||||
evidence_refs: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
proposed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
# Model-claimed executed actions are informational only — the fact ledger
|
||||
# never trusts them; ActionCoordinator is the sole source of truth.
|
||||
executed_actions: list[str | dict[str, Any]] = Field(default_factory=list)
|
||||
open_questions: list[str] = Field(default_factory=list)
|
||||
conflicts: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ConfirmedCandidate(BaseModel):
|
||||
"""A candidate fact the cross-review confirms, WITH required evidence.
|
||||
|
||||
The server only promotes a candidate to confirmed_facts when: the
|
||||
candidate_id exists in candidate_facts AND at least one evidence_ref is
|
||||
supplied AND every evidence_ref resolves to a known evidence/tool result in
|
||||
the ledger. This prevents a weak model from promoting a bare seat claim to
|
||||
an "established fact" with no backing.
|
||||
"""
|
||||
|
||||
candidate_id: str = Field(..., min_length=1)
|
||||
evidence_refs: list[str] = Field(..., min_length=1)
|
||||
|
||||
|
||||
class CrossReviewOutput(BaseModel):
|
||||
"""Validated output of the cross-review stage."""
|
||||
|
||||
summary: str = Field(..., min_length=1)
|
||||
# Conservative default: a review must positively assert evidence is OK; we
|
||||
# never assume it. (Previously defaulted True, which let a model that
|
||||
# omitted the field count as "evidence sufficient".)
|
||||
evidence_ok: bool = False
|
||||
gaps: list[str] = Field(default_factory=list)
|
||||
conflicts: list[str] = Field(default_factory=list)
|
||||
tool_failures: list[str] = Field(default_factory=list)
|
||||
write_status_notes: list[str] = Field(default_factory=list)
|
||||
recommendations: list[str] = Field(default_factory=list)
|
||||
# Phase-3 P1-1: confirmed candidates now carry mandatory evidence. The
|
||||
# legacy bare-id list is still accepted (for back-compat with old prompts)
|
||||
# but is treated as "no evidence" and therefore NOT promoted.
|
||||
confirmed_candidates: list[ConfirmedCandidate] = Field(default_factory=list)
|
||||
confirmed_candidate_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_json_object(text: str) -> str | None:
|
||||
"""Best-effort extract of a JSON object from model text."""
|
||||
raw = (text or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
m = _JSON_FENCE_RE.search(raw)
|
||||
if m:
|
||||
raw = m.group(1).strip()
|
||||
if raw.startswith("{") and raw.endswith("}"):
|
||||
return raw
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
return raw[start : end + 1]
|
||||
return None
|
||||
|
||||
|
||||
def parse_seat_output(text: str) -> tuple[SeatTurnOutput | None, list[str]]:
|
||||
"""Parse seat text into SeatTurnOutput. Returns (model, readable_errors)."""
|
||||
blob = _extract_json_object(text)
|
||||
if blob is None:
|
||||
return None, ["output is not a JSON object; emit a single JSON object matching SeatTurnOutput"]
|
||||
try:
|
||||
data = json.loads(blob)
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, [f"invalid JSON: {exc.msg}"]
|
||||
if not isinstance(data, dict):
|
||||
return None, ["JSON root must be an object"]
|
||||
try:
|
||||
return SeatTurnOutput.model_validate(data), []
|
||||
except ValidationError as exc:
|
||||
errors = []
|
||||
for err in exc.errors():
|
||||
loc = ".".join(str(p) for p in err.get("loc", ()))
|
||||
errors.append(f"{loc}: {err.get('msg', 'invalid')}")
|
||||
return None, errors or ["schema validation failed"]
|
||||
|
||||
|
||||
def parse_cross_review_output(text: str) -> tuple[CrossReviewOutput | None, list[str]]:
|
||||
blob = _extract_json_object(text)
|
||||
if blob is None:
|
||||
return None, ["output is not a JSON object; emit CrossReviewOutput JSON"]
|
||||
try:
|
||||
data = json.loads(blob)
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, [f"invalid JSON: {exc.msg}"]
|
||||
if not isinstance(data, dict):
|
||||
return None, ["JSON root must be an object"]
|
||||
try:
|
||||
return CrossReviewOutput.model_validate(data), []
|
||||
except ValidationError as exc:
|
||||
errors = [
|
||||
f"{'.'.join(str(p) for p in e.get('loc', ()))}: {e.get('msg', 'invalid')}" for e in exc.errors()
|
||||
]
|
||||
return None, errors or ["schema validation failed"]
|
||||
|
||||
|
||||
def build_repair_prompt(raw: str, errors: list[str], *, schema_name: str = "SeatTurnOutput") -> str:
|
||||
"""Build a one-shot repair instruction for the seat."""
|
||||
err_lines = "\n".join(f"- {e}" for e in errors[:12])
|
||||
return (
|
||||
f"你上一次输出未通过 {schema_name} 校验。请只输出一个 JSON 对象(不要 Markdown 说明),"
|
||||
f"修正下列问题后重试一次:\n{err_lines}\n\n"
|
||||
f"原始输出(供参考,请重写为合法 JSON):\n{raw[:4000]}"
|
||||
)
|
||||
|
||||
|
||||
def seat_output_instruction() -> str:
|
||||
"""Stage instruction fragment telling the model to emit SeatTurnOutput JSON."""
|
||||
return (
|
||||
"请以 JSON 对象回复,字段:summary(string,必填)、findings(string[])、"
|
||||
"evidence_refs、proposed_actions、executed_actions、open_questions、conflicts。"
|
||||
"proposed_actions 只表示建议;不要把未真正由系统执行的外部写入标为已执行。"
|
||||
)
|
||||
|
||||
|
||||
async def validate_or_repair_seat_output(
|
||||
*,
|
||||
raw_content: str,
|
||||
repair_runner: RepairRunner | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate seat output; optionally run one repair turn via ``repair_runner``.
|
||||
|
||||
``repair_runner`` is an async callable ``(repair_prompt: str) -> str``.
|
||||
|
||||
Returns::
|
||||
{
|
||||
"schema_status": "ok" | "schema_invalid" | "repaired",
|
||||
"output": dict | None,
|
||||
"errors": [...],
|
||||
"raw_content": str,
|
||||
"repair_attempted": bool,
|
||||
}
|
||||
"""
|
||||
model, errors = parse_seat_output(raw_content)
|
||||
if model is not None:
|
||||
return {
|
||||
"schema_status": "ok",
|
||||
"output": model.model_dump(),
|
||||
"errors": [],
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": False,
|
||||
}
|
||||
|
||||
if repair_runner is None:
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors,
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": False,
|
||||
}
|
||||
|
||||
repair_prompt = build_repair_prompt(raw_content, errors)
|
||||
try:
|
||||
repaired_text = await repair_runner(repair_prompt)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors + [f"repair failed: {type(exc).__name__}"],
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": True,
|
||||
}
|
||||
|
||||
model2, errors2 = parse_seat_output(str(repaired_text or ""))
|
||||
if model2 is not None:
|
||||
return {
|
||||
"schema_status": "repaired",
|
||||
"output": model2.model_dump(),
|
||||
"errors": [],
|
||||
"raw_content": str(repaired_text),
|
||||
"repair_attempted": True,
|
||||
}
|
||||
return {
|
||||
"schema_status": "schema_invalid",
|
||||
"output": None,
|
||||
"errors": errors2 or errors,
|
||||
"raw_content": raw_content,
|
||||
"repair_attempted": True,
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,7 +5,8 @@ The SDK is imported lazily so the Gateway starts without it when the feature
|
||||
flag is off.
|
||||
|
||||
Two topologies:
|
||||
- ``structured``: coordinator → parallel seats → approval gate → finalizer
|
||||
- ``structured`` (Phase 3): planner → parallel experts → cross-review →
|
||||
approval gate → finalizer
|
||||
- ``group_chat``: coordinator loop (dynamic speaker selection) → finalizer
|
||||
|
||||
Each node is a real MAF ``Executor`` subclass instance (from
|
||||
@@ -24,40 +25,42 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def build_structured_workflow(
|
||||
*,
|
||||
coordinator: Any,
|
||||
planner: Any,
|
||||
seat_executors: list[Any],
|
||||
cross_review: Any,
|
||||
approval_gate: Any,
|
||||
finalizer: Any,
|
||||
waiting_terminal: Any,
|
||||
) -> Any:
|
||||
"""Build a structured (fixed-topology) MAF Workflow.
|
||||
|
||||
Topology: coordinator → parallel seat executors → approval gate → finalizer.
|
||||
The approval gate conditionally routes to waiting_terminal (pause) or finalizer.
|
||||
Topology: planner → parallel seat executors → cross_review → approval gate
|
||||
→ finalizer | waiting_terminal.
|
||||
"""
|
||||
from agent_framework import Case, Default, WorkflowBuilder
|
||||
|
||||
builder = WorkflowBuilder(
|
||||
start_executor=coordinator,
|
||||
start_executor=planner,
|
||||
name="maf-structured-roundtable",
|
||||
output_from=[finalizer, waiting_terminal],
|
||||
)
|
||||
|
||||
# coordinator → seats → approval gate. Handle 0/1/≥2 seats since the SDK's
|
||||
# FanOut/FanIn edge groups require at least two members.
|
||||
if len(seat_executors) >= 2:
|
||||
builder.add_fan_out_edges(coordinator, seat_executors)
|
||||
builder.add_fan_in_edges(seat_executors, approval_gate)
|
||||
builder.add_fan_out_edges(planner, seat_executors)
|
||||
builder.add_fan_in_edges(seat_executors, cross_review)
|
||||
elif len(seat_executors) == 1:
|
||||
builder.add_edge(coordinator, seat_executors[0])
|
||||
builder.add_edge(seat_executors[0], approval_gate)
|
||||
builder.add_edge(planner, seat_executors[0])
|
||||
builder.add_edge(seat_executors[0], cross_review)
|
||||
else:
|
||||
# No seats — coordinator feeds the approval gate directly.
|
||||
builder.add_edge(coordinator, approval_gate)
|
||||
builder.add_edge(planner, cross_review)
|
||||
|
||||
builder.add_edge(cross_review, approval_gate)
|
||||
|
||||
# approval gate → conditional: approved → finalizer, waiting → waiting_terminal
|
||||
builder.add_switch_case_edge_group(approval_gate, [
|
||||
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
|
||||
Case(
|
||||
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
|
||||
target=waiting_terminal,
|
||||
),
|
||||
Default(target=finalizer),
|
||||
])
|
||||
|
||||
@@ -86,24 +89,23 @@ def build_group_chat_workflow(
|
||||
output_from=[finalizer, waiting_terminal],
|
||||
)
|
||||
|
||||
# coordinator → switch-case: route to the selected seat, or to approval gate
|
||||
cases = []
|
||||
for i, agent_id in enumerate(seat_agent_ids):
|
||||
cases.append(Case(
|
||||
condition=lambda out, aid=agent_id: isinstance(out, dict) and out.get("agent_id") == aid,
|
||||
target=seat_executors[i],
|
||||
))
|
||||
# Default: coordinator said "done" → go to approval gate
|
||||
cases.append(Default(target=approval_gate))
|
||||
builder.add_switch_case_edge_group(coordinator, cases)
|
||||
|
||||
# each seat → back to coordinator (loop)
|
||||
for seat_exec in seat_executors:
|
||||
builder.add_edge(seat_exec, coordinator)
|
||||
|
||||
# approval gate → conditional: waiting → waiting_terminal, else → finalizer
|
||||
builder.add_switch_case_edge_group(approval_gate, [
|
||||
Case(condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval", target=waiting_terminal),
|
||||
Case(
|
||||
condition=lambda out: isinstance(out, dict) and out.get("decision") == "waiting_approval",
|
||||
target=waiting_terminal,
|
||||
),
|
||||
Default(target=finalizer),
|
||||
])
|
||||
|
||||
@@ -113,15 +115,19 @@ def build_group_chat_workflow(
|
||||
def create_workflow(
|
||||
*,
|
||||
mode: str,
|
||||
coordinator: Any,
|
||||
seat_executors: list[Any],
|
||||
approval_gate: Any,
|
||||
finalizer: Any,
|
||||
waiting_terminal: Any,
|
||||
seat_agent_ids: list[str],
|
||||
coordinator: Any | None = None,
|
||||
planner: Any | None = None,
|
||||
cross_review: Any | None = None,
|
||||
) -> Any:
|
||||
"""Factory: create the appropriate workflow for the given mode."""
|
||||
if mode == "group_chat":
|
||||
if coordinator is None:
|
||||
raise ValueError("group_chat mode requires coordinator")
|
||||
return build_group_chat_workflow(
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
@@ -130,9 +136,12 @@ def create_workflow(
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=seat_agent_ids,
|
||||
)
|
||||
if planner is None or cross_review is None:
|
||||
raise ValueError("structured mode requires planner and cross_review")
|
||||
return build_structured_workflow(
|
||||
coordinator=coordinator,
|
||||
planner=planner,
|
||||
seat_executors=seat_executors,
|
||||
cross_review=cross_review,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
|
||||
@@ -23,12 +23,15 @@ from typing import Any
|
||||
|
||||
from app.maf_roundtable.action_coordinator import ActionCoordinator
|
||||
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter
|
||||
from app.maf_roundtable.fact_ledger import record_action_transition
|
||||
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
||||
from app.maf_roundtable.sanitize import sanitize
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CoordinatorExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
WaitingTerminalExecutor,
|
||||
@@ -92,6 +95,7 @@ class WorkflowRunner:
|
||||
|
||||
seat_agents = [p["agent_id"] for p in participants if p.get("role") != "manager"]
|
||||
manager_id = session.get("manager_agent_id")
|
||||
seat_timeout, schema_repair = self._orchestration_settings()
|
||||
|
||||
state = RoundtableState(
|
||||
session_id=session_id,
|
||||
@@ -106,28 +110,11 @@ class WorkflowRunner:
|
||||
seat_agent_ids=seat_agents,
|
||||
thread_map=thread_map,
|
||||
capability_snapshot=session.get("capability_snapshot"),
|
||||
seat_timeout_seconds=seat_timeout,
|
||||
schema_repair_enabled=schema_repair,
|
||||
)
|
||||
|
||||
# Build executor instances — one per seat, each bound to a fixed agent_id.
|
||||
coordinator = CoordinatorExecutor(self._adapter, state)
|
||||
seat_executors = [
|
||||
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
||||
for agent_id in seat_agents
|
||||
]
|
||||
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
||||
finalizer = FinalizerExecutor(self._adapter, state)
|
||||
waiting_terminal = WaitingTerminalExecutor()
|
||||
|
||||
workflow = create_workflow(
|
||||
mode=state.mode,
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=seat_agents,
|
||||
)
|
||||
|
||||
workflow = self._build_workflow(state)
|
||||
await self._run_workflow(workflow, state)
|
||||
|
||||
# Persist final state. waiting_approval is NOT a terminal state —
|
||||
@@ -162,35 +149,103 @@ class WorkflowRunner:
|
||||
except Exception as exc:
|
||||
await self._fail_run(run_id, session_id, t0, type(exc).__name__, str(exc))
|
||||
|
||||
def _orchestration_settings(self) -> tuple[float, bool]:
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
cfg = get_app_config().maf_roundtable
|
||||
return (
|
||||
float(getattr(cfg, "seat_timeout_seconds", 600) or 600),
|
||||
bool(getattr(cfg, "schema_repair_enabled", True)),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return 600.0, True
|
||||
|
||||
def _build_workflow(self, state: RoundtableState) -> Any:
|
||||
seat_executors = [
|
||||
SeatExecutor(self._adapter, state, agent_id, action_coordinator=self._action_coordinator)
|
||||
for agent_id in state.seat_agent_ids
|
||||
]
|
||||
approval_gate = ApprovalGateExecutor(self._adapter, state, self._service)
|
||||
finalizer = FinalizerExecutor(self._adapter, state)
|
||||
waiting_terminal = WaitingTerminalExecutor()
|
||||
|
||||
if state.mode == "group_chat":
|
||||
coordinator = CoordinatorExecutor(self._adapter, state)
|
||||
return create_workflow(
|
||||
mode=state.mode,
|
||||
coordinator=coordinator,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=state.seat_agent_ids,
|
||||
)
|
||||
|
||||
planner = PlannerExecutor(self._adapter, state)
|
||||
cross_review = CrossReviewExecutor(self._adapter, state)
|
||||
return create_workflow(
|
||||
mode="structured",
|
||||
planner=planner,
|
||||
cross_review=cross_review,
|
||||
seat_executors=seat_executors,
|
||||
approval_gate=approval_gate,
|
||||
finalizer=finalizer,
|
||||
waiting_terminal=waiting_terminal,
|
||||
seat_agent_ids=state.seat_agent_ids,
|
||||
)
|
||||
|
||||
async def _persist_checkpoint(self, state: RoundtableState, *, reason: str = "") -> None:
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
||||
|
||||
await self._adapter._emit_critical(
|
||||
state.run_id,
|
||||
make_workflow_checkpointed(
|
||||
None,
|
||||
state.current_round,
|
||||
checkpoint_id=f"nodes-{len(state.completed_nodes)}",
|
||||
reason=reason or f"stage={state.stage}",
|
||||
),
|
||||
)
|
||||
|
||||
async def _run_workflow(self, workflow: Any, state: RoundtableState) -> None:
|
||||
"""Execute the workflow graph via the MAF SDK's streaming API.
|
||||
|
||||
Consumes ``WorkflowEvent``s and translates them into MAF roundtable
|
||||
events. Persists the RoundtableState checkpoint after each superstep.
|
||||
"""
|
||||
from app.maf_roundtable.event_normalizer import make_workflow_checkpointed
|
||||
|
||||
async for event in workflow.run(state.task_text, stream=True, include_status_events=True):
|
||||
event_type = getattr(event, "type", "")
|
||||
executor_id = getattr(event, "executor_id", None)
|
||||
|
||||
if event_type == "superstep_completed":
|
||||
# Persist checkpoint after each superstep.
|
||||
iteration = getattr(event, "iteration", None)
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
await self._adapter._emit_critical(state.run_id, make_workflow_checkpointed(
|
||||
executor_id, state.current_round,
|
||||
checkpoint_id=f"superstep-{iteration}",
|
||||
reason="superstep_completed",
|
||||
))
|
||||
await self._persist_checkpoint(
|
||||
state, reason=f"superstep_completed:{iteration}"
|
||||
)
|
||||
|
||||
elif event_type in ("executor_completed", "executor_invoked"):
|
||||
# Persist after each node so completed_nodes survive restarts.
|
||||
if state.completed_nodes:
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
|
||||
elif event_type == "executor_failed":
|
||||
details = getattr(event, "details", None)
|
||||
error_msg = str(details) if details else "executor failed"
|
||||
# Soft seat failures are handled inside SeatExecutor; only hard
|
||||
# failures (e.g. WriteGuardConfigurationError) abort the run.
|
||||
raise RuntimeError(f"MAF executor '{executor_id}' failed: {error_msg}")
|
||||
|
||||
# Always persist latest completed_nodes at end of stream.
|
||||
await self._service.store.runs.update_run(state.run_id, {
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
})
|
||||
|
||||
# Check if the workflow ended in a waiting_approval state.
|
||||
if state.status == "waiting_approval" or state.pending_approvals:
|
||||
state.status = "waiting_approval"
|
||||
@@ -286,10 +341,22 @@ class WorkflowRunner:
|
||||
for pa in state.pending_approvals:
|
||||
approval_id = pa.get("approval_id", "")
|
||||
action_id = pa.get("action_id", "")
|
||||
# Extracted up-front so EVERY branch (rejected / failed / approved)
|
||||
# can write the ledger transition with the correct agent_id.
|
||||
agent_id = pa.get("agent_id", "")
|
||||
thread_id = state.thread_map.get(agent_id, "")
|
||||
tool_name = pa.get("tool_name", "")
|
||||
approval = await self._service.store.approvals.get_approval(approval_id)
|
||||
if approval is None:
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": "approval not found"})
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="failed",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
approval_status = approval.get("status", "pending")
|
||||
@@ -307,17 +374,32 @@ class WorkflowRunner:
|
||||
decided_by=approval.get("decided_by", ""),
|
||||
))
|
||||
action_outcomes.append({"action_id": action_id, "status": "rejected"})
|
||||
# Phase-3 P1-3: move the ledger row out of pending into the
|
||||
# failed/unknown partition as rejected.
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="rejected",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if approval_status != "approved":
|
||||
action_outcomes.append({"action_id": action_id, "status": "failed",
|
||||
"error": f"unexpected approval status: {approval_status}"})
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=pa.get("tool_name", ""),
|
||||
status="failed",
|
||||
agent_id=agent_id,
|
||||
)
|
||||
continue
|
||||
|
||||
# Approved — execute deterministically via ToolExecutor.
|
||||
agent_id = pa.get("agent_id", "")
|
||||
thread_id = state.thread_map.get(agent_id, "")
|
||||
tool_name = pa.get("tool_name", "")
|
||||
# (agent_id / thread_id / tool_name are already extracted at the
|
||||
# top of this loop iteration so every branch can use them.)
|
||||
|
||||
# Conditional transition (waiting_approval → approved). The
|
||||
# authoritative at-most-once gate is the ToolExecutor's
|
||||
@@ -378,11 +460,20 @@ class WorkflowRunner:
|
||||
"error": outcome.get("error"),
|
||||
})
|
||||
|
||||
state.fact_ledger["pending_actions"].append({
|
||||
"action_id": action_id,
|
||||
"tool_name": tool_name,
|
||||
"outcome": outcome.get("status"),
|
||||
})
|
||||
out_status = outcome.get("status")
|
||||
ledger_status = {
|
||||
"succeeded": "executed",
|
||||
"failed": "failed",
|
||||
"unknown": "unknown",
|
||||
"skipped": "executed",
|
||||
}.get(str(out_status), "unknown")
|
||||
record_action_transition(
|
||||
state.fact_ledger,
|
||||
action_id=action_id,
|
||||
tool_name=tool_name,
|
||||
status=ledger_status,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
|
||||
# P0-3: determine parent run status from action outcomes.
|
||||
statuses = {o.get("status") for o in action_outcomes}
|
||||
@@ -451,6 +542,76 @@ class WorkflowRunner:
|
||||
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
async def recover(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""Graph-level recovery after process restart (MAF-03-07).
|
||||
|
||||
Reloads ``RoundtableState`` from ``maf_checkpoint`` and re-enters the
|
||||
structured workflow. Executors whose ids are in ``completed_nodes``
|
||||
short-circuit without calling DeerFlow again.
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
import agent_framework # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise MafSdkMissingError("agent_framework SDK is not installed") from exc
|
||||
|
||||
run = await self._service.store.runs.get_run(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] not in ("running", "waiting_approval"):
|
||||
logger.info("recover(%s): status=%s — nothing to recover", run_id, run["status"])
|
||||
return run
|
||||
|
||||
checkpoint = run.get("maf_checkpoint")
|
||||
if not checkpoint:
|
||||
logger.warning("recover(%s): no checkpoint", run_id)
|
||||
return run
|
||||
|
||||
state = RoundtableState.from_dict(checkpoint)
|
||||
session = await self._service.store.sessions.get_session(state.session_id, user_id=None)
|
||||
if session is None:
|
||||
return run
|
||||
|
||||
# Ensure threads still bound.
|
||||
participants = await self._service.store.participants.list_participants(state.session_id)
|
||||
thread_map = await self._adapter.ensure_threads(state.session_id, run_id, participants)
|
||||
state.thread_map.update(thread_map)
|
||||
|
||||
if run["status"] == "waiting_approval":
|
||||
# Approval path stays on the existing resume() — recover only
|
||||
# continues orchestration for running graphs.
|
||||
return run
|
||||
|
||||
await self._service.store.runs.update_run(run_id, {"status": "running"})
|
||||
try:
|
||||
workflow = self._build_workflow(state)
|
||||
await self._run_workflow(workflow, state)
|
||||
|
||||
latency = int((time.perf_counter() - t0) * 1000)
|
||||
final_status = state.status
|
||||
update_fields: dict[str, Any] = {
|
||||
"status": final_status,
|
||||
"latency_ms": latency,
|
||||
"maf_checkpoint": state.to_dict(),
|
||||
"token_usage": _aggregate_token_usage(state.turn_results),
|
||||
}
|
||||
if final_status in ("completed", "failed", "cancelled"):
|
||||
update_fields["finished_at"] = datetime.now(UTC)
|
||||
await self._service.store.runs.update_run(run_id, update_fields)
|
||||
|
||||
session_fields: dict[str, Any] = {"status": final_status}
|
||||
if final_status == "completed":
|
||||
session_fields["final_result"] = state.final_result
|
||||
if final_status != "waiting_approval":
|
||||
session_fields["fact_ledger"] = state.fact_ledger
|
||||
await self._service.store.sessions.update_session(
|
||||
state.session_id, user_id=self._user_id, fields=session_fields
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
await self._fail_run(run_id, state.session_id, t0, type(exc).__name__, str(exc))
|
||||
|
||||
return await self._service.store.runs.get_run(run_id)
|
||||
|
||||
|
||||
def _aggregate_token_usage(turn_results: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Aggregate token usage across all seat turns into a run-level summary."""
|
||||
|
||||
@@ -563,6 +563,8 @@ maf_roundtable:
|
||||
default_write_policy: confirm_high_risk # auto | confirm_high_risk | confirm_all_writes
|
||||
default_max_rounds: 8 # 群聊模式轮次硬上限
|
||||
admin_diagnostic_access: true # 管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)
|
||||
seat_timeout_seconds: 600 # 结构化流程单席位超时(秒)
|
||||
schema_repair_enabled: true # 席位 JSON 校验失败时允许一次定向修复
|
||||
|
||||
# ── WriteGuard 中间件配置 ────────────────────────────────────────────────────
|
||||
# 与 maf_roundtable 联动:MAF 会商运行时,WriteGuard 中间件拦截外部写工具调用,
|
||||
|
||||
+12
@@ -62,6 +62,18 @@ class MafRoundtableConfig(BaseModel):
|
||||
default=True,
|
||||
description="管理员是否可跨用户读取所有 MAF 会商用于诊断(仅读,不含写)",
|
||||
)
|
||||
# Phase 3: per-seat wall-clock timeout (seconds) for structured expert turns.
|
||||
seat_timeout_seconds: int = Field(
|
||||
default=600,
|
||||
ge=30,
|
||||
le=7200,
|
||||
description="结构化流程中单个专家席位的超时秒数;超时记为 partial_failure 并继续其余席位",
|
||||
)
|
||||
# Phase 3: allow one directed format-repair turn when seat JSON fails schema.
|
||||
schema_repair_enabled: bool = Field(
|
||||
default=True,
|
||||
description="席位输出未通过 SeatTurnOutput 校验时是否允许一次定向格式修复",
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+37
-3
@@ -30,6 +30,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
@@ -306,13 +307,34 @@ class MafRoundtableSeatRunRow(Base):
|
||||
the system can still locate and cancel outstanding DeerFlow runs, and the
|
||||
MAF checkpoint can reference which seat turns completed.
|
||||
|
||||
Unique constraint: ``(maf_run_id, agent_id, turn_index)`` — one DeerFlow run
|
||||
per seat per turn.
|
||||
Phase-3 unique constraint: ``(maf_run_id, agent_id, turn_index,
|
||||
execution_stage)``. ``turn_index`` is the business round (kept stable for
|
||||
frontend display / approval matching / event tracing); ``execution_stage``
|
||||
disambiguates the distinct DeerFlow executions that happen within one
|
||||
business round (planner / expert / format_repair / cross_review /
|
||||
finalizer / group_chat). The business turn_index is NEVER mutated to dodge
|
||||
a collision.
|
||||
"""
|
||||
|
||||
__tablename__ = "maf_roundtable_seat_runs"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("maf_run_id", "agent_id", "turn_index", name="uq_maf_seat_runs_run_agent_turn"),
|
||||
UniqueConstraint(
|
||||
"maf_run_id",
|
||||
"agent_id",
|
||||
"turn_index",
|
||||
"execution_stage",
|
||||
name="uq_maf_seat_runs_run_agent_turn_stage",
|
||||
),
|
||||
# Phase-3 P1-2: enforce the closed execution_stage enum at the DB layer.
|
||||
# Mirrors seat_runs.VALID_EXECUTION_STAGES — a typo (e.g. "exprt") would
|
||||
# otherwise silently bypass the four-column idempotency key by creating
|
||||
# a brand-new stage. This CHECK keeps both fresh creates and any raw
|
||||
# migration / manual SQL honest. Keep in sync with VALID_EXECUTION_STAGES.
|
||||
CheckConstraint(
|
||||
"execution_stage IN ('legacy','planner','expert','format_repair',"
|
||||
"'cross_review','cross_review_repair','finalizer','group_chat')",
|
||||
name="ck_maf_seat_runs_execution_stage",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
@@ -321,12 +343,24 @@ class MafRoundtableSeatRunRow(Base):
|
||||
)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
turn_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
# Phase-3: which kind of DeerFlow execution this row represents. Pre-Phase-3
|
||||
# rows are migrated to ``legacy``. Closed enum (see seat_runs.VALID_EXECUTION_STAGES):
|
||||
# planner | expert | format_repair | cross_review | cross_review_repair |
|
||||
# finalizer | group_chat | legacy.
|
||||
execution_stage: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="legacy", server_default="legacy"
|
||||
)
|
||||
deerflow_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
deerflow_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
sandbox_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
workflow_checkpoint_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# pending | running | completed | failed | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
|
||||
# Phase-3 P0-1: a sanitized, replayable snapshot of the SeatResponse written
|
||||
# atomically with the completed transition. Survives a crash that happens
|
||||
# after the seat run is marked completed but before the workflow checkpoint
|
||||
# is persisted, so recovery never needs to re-call DeerFlow.
|
||||
result_snapshot: Mapped[dict | None] = mapped_column(PortableJSON, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
|
||||
+249
-8
@@ -12,6 +12,7 @@ from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.maf_roundtable.model import MafRoundtableSeatRunRow
|
||||
@@ -21,17 +22,68 @@ def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
# ── execution_stage enum (Phase-3 P1-2) ─────────────────────────────────────
|
||||
# The fixed set of distinct DeerFlow executions that may occur within one
|
||||
# business round (turn_index). The four-column idempotency key
|
||||
# (maf_run_id, agent_id, turn_index, execution_stage) only works if stages
|
||||
# are a closed set — a typo would silently create a new stage and bypass the
|
||||
# dedup. Validate on write so a misspelling raises loudly instead.
|
||||
LEGACY = "legacy"
|
||||
PLANNER = "planner"
|
||||
EXPERT = "expert"
|
||||
FORMAT_REPAIR = "format_repair"
|
||||
CROSS_REVIEW = "cross_review"
|
||||
CROSS_REVIEW_REPAIR = "cross_review_repair"
|
||||
FINALIZER = "finalizer"
|
||||
GROUP_CHAT = "group_chat"
|
||||
|
||||
VALID_EXECUTION_STAGES: frozenset[str] = frozenset({
|
||||
LEGACY,
|
||||
PLANNER,
|
||||
EXPERT,
|
||||
FORMAT_REPAIR,
|
||||
CROSS_REVIEW,
|
||||
CROSS_REVIEW_REPAIR,
|
||||
FINALIZER,
|
||||
GROUP_CHAT,
|
||||
})
|
||||
|
||||
|
||||
class InvalidExecutionStageError(ValueError):
|
||||
"""Raised when an execution_stage outside the closed enum is written."""
|
||||
|
||||
|
||||
class SeatRunNotFoundError(LookupError):
|
||||
"""Raised when a seat run expected by the four-tuple does not exist."""
|
||||
|
||||
|
||||
class SeatRunWaitTimeout(TimeoutError):
|
||||
"""Raised when waiting for a seat run to reach a terminal state times out."""
|
||||
|
||||
|
||||
def _validate_stage(execution_stage: str) -> str:
|
||||
if execution_stage not in VALID_EXECUTION_STAGES:
|
||||
raise InvalidExecutionStageError(
|
||||
f"execution_stage={execution_stage!r} is not one of the allowed "
|
||||
f"stages {sorted(VALID_EXECUTION_STAGES)}; a typo here would bypass "
|
||||
f"the four-column idempotency key."
|
||||
)
|
||||
return execution_stage
|
||||
|
||||
|
||||
def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"maf_run_id": row.maf_run_id,
|
||||
"agent_id": row.agent_id,
|
||||
"turn_index": row.turn_index,
|
||||
"execution_stage": row.execution_stage,
|
||||
"deerflow_run_id": row.deerflow_run_id,
|
||||
"deerflow_thread_id": row.deerflow_thread_id,
|
||||
"sandbox_id": row.sandbox_id,
|
||||
"workflow_checkpoint_id": row.workflow_checkpoint_id,
|
||||
"status": row.status,
|
||||
"result_snapshot": row.result_snapshot,
|
||||
"started_at": row.started_at.isoformat() if isinstance(row.started_at, datetime) else row.started_at,
|
||||
"finished_at": row.finished_at.isoformat() if isinstance(row.finished_at, datetime) else row.finished_at,
|
||||
"error": row.error,
|
||||
@@ -40,6 +92,12 @@ def _to_dict(row: MafRoundtableSeatRunRow) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
# Seat-run lifecycle statuses that mean "still in flight" — a second attempt to
|
||||
# start the same DeerFlow execution must NOT launch another run while one of
|
||||
# these exists.
|
||||
_ACTIVE_STATUSES = ("pending", "running")
|
||||
|
||||
|
||||
class MafSeatRunRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -50,14 +108,17 @@ class MafSeatRunRepository:
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str = "legacy",
|
||||
deerflow_thread_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
_validate_stage(execution_stage)
|
||||
now = _now()
|
||||
row = MafRoundtableSeatRunRow(
|
||||
id=uuid4().hex,
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
deerflow_thread_id=deerflow_thread_id,
|
||||
status="pending",
|
||||
created_at=now,
|
||||
@@ -68,6 +129,171 @@ class MafSeatRunRepository:
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
async def get_by_four_tuple(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Look up a seat run by its Phase-3 identity key (no DeerFlow side effects)."""
|
||||
stmt = select(MafRoundtableSeatRunRow).where(
|
||||
MafRoundtableSeatRunRow.maf_run_id == maf_run_id,
|
||||
MafRoundtableSeatRunRow.agent_id == agent_id,
|
||||
MafRoundtableSeatRunRow.turn_index == turn_index,
|
||||
MafRoundtableSeatRunRow.execution_stage == execution_stage,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def wait_for_terminal_seat_run(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
timeout_seconds: float,
|
||||
poll_interval_seconds: float = 0.1,
|
||||
) -> dict[str, Any]:
|
||||
"""Block until the seat run identified by the four-tuple reaches a terminal status.
|
||||
|
||||
Terminal statuses: completed, failed, cancelled, cancel_failed, cancel_unknown.
|
||||
Non-terminal (still in-flight): pending, running, cancelling.
|
||||
|
||||
Raises:
|
||||
SeatRunNotFoundError: if no row exists for the four-tuple.
|
||||
SeatRunWaitTimeout: if the deadline is reached while still non-terminal.
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
_TERMINAL = ("completed", "failed", "cancelled", "cancel_failed", "cancel_unknown")
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
|
||||
while True:
|
||||
row = await self.get_by_four_tuple(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if row is None:
|
||||
raise SeatRunNotFoundError(
|
||||
f"seat run not found: maf_run_id={maf_run_id!r} agent_id={agent_id!r} "
|
||||
f"turn_index={turn_index} execution_stage={execution_stage!r}"
|
||||
)
|
||||
if row["status"] in _TERMINAL:
|
||||
return row
|
||||
if time.monotonic() >= deadline:
|
||||
raise SeatRunWaitTimeout(
|
||||
f"timed out waiting for seat run {row['id']!r} "
|
||||
f"(stage={execution_stage!r}, status={row['status']!r}) "
|
||||
f"after {timeout_seconds}s"
|
||||
)
|
||||
await asyncio.sleep(poll_interval_seconds)
|
||||
|
||||
async def get_or_create_seat_run(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
deerflow_thread_id: str | None = None,
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""Atomic idempotent seat-run acquisition (Phase-3 P0-1).
|
||||
|
||||
Race-free: tries to INSERT first; on a four-column unique-key conflict
|
||||
it rolls back the short transaction and reads the existing row. This is
|
||||
strictly stronger than "query-then-insert" because two concurrent
|
||||
callers that both miss the lookup would otherwise both insert — only
|
||||
the DB constraint decides the winner, and the loser reads the winner's
|
||||
row.
|
||||
|
||||
Because the winner's transaction may not have committed by the time the
|
||||
loser's post-conflict read runs (session isolation), the loser retries
|
||||
the read a bounded number of times with a short backoff before giving
|
||||
up. ``IntegrityError`` is NEVER propagated to the caller on a four-tuple
|
||||
conflict.
|
||||
|
||||
Returns ``(seat_run, created)``:
|
||||
- ``created=True`` → caller OWNS this execution and may start DeerFlow.
|
||||
- ``created=False`` → a row already exists; the caller must inspect its
|
||||
``status`` and must NOT start a second DeerFlow run for it.
|
||||
"""
|
||||
_validate_stage(execution_stage)
|
||||
now = _now()
|
||||
row = MafRoundtableSeatRunRow(
|
||||
id=uuid4().hex,
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
deerflow_thread_id=deerflow_thread_id,
|
||||
status="pending",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Lost the race (or replayed an existing four-tuple). Roll back
|
||||
# this short transaction, then read the winner's row. The winner
|
||||
# may not have committed yet (session isolation), so retry a few
|
||||
# times with a short backoff before treating this as a hard
|
||||
# error.
|
||||
await session.rollback()
|
||||
existing = await self._await_existing(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if existing is None:
|
||||
# The conflict was NOT on our four-tuple (e.g. a FK
|
||||
# violation) — surface it as a clean error rather than
|
||||
# swallowing it.
|
||||
raise
|
||||
return existing, False
|
||||
return _to_dict(row), True
|
||||
|
||||
async def _await_existing(
|
||||
self,
|
||||
*,
|
||||
maf_run_id: str,
|
||||
agent_id: str,
|
||||
turn_index: int,
|
||||
execution_stage: str,
|
||||
retries: int = 10,
|
||||
backoff_s: float = 0.01,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Read the winner's row, retrying briefly until it is visible.
|
||||
|
||||
Under true concurrency the winner's COMMIT may land after the loser's
|
||||
post-rollback SELECT; the loser cannot see an uncommitted row, so we
|
||||
poll with a tiny backoff until the winner's transaction commits (or the
|
||||
retry budget is exhausted, in which case we return None so the caller
|
||||
can decide it was a non-four-tuple conflict).
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
for _ in range(max(1, retries)):
|
||||
existing = await self.get_by_four_tuple(
|
||||
maf_run_id=maf_run_id,
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
execution_stage=execution_stage,
|
||||
)
|
||||
if existing is not None:
|
||||
return existing
|
||||
await asyncio.sleep(backoff_s)
|
||||
return None
|
||||
|
||||
async def mark_running(
|
||||
self, seat_run_id: str, *, deerflow_run_id: str, sandbox_id: str | None = None
|
||||
) -> None:
|
||||
@@ -85,17 +311,29 @@ class MafSeatRunRepository:
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def mark_completed(self, seat_run_id: str, *, checkpoint_id: str | None = None) -> None:
|
||||
async def mark_completed(
|
||||
self,
|
||||
seat_run_id: str,
|
||||
*,
|
||||
checkpoint_id: str | None = None,
|
||||
result_snapshot: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Mark a seat run completed, optionally persisting a sanitized result
|
||||
snapshot so it can be replayed after a crash even if the workflow
|
||||
checkpoint was not yet written."""
|
||||
values: dict[str, Any] = {
|
||||
"status": "completed",
|
||||
"workflow_checkpoint_id": checkpoint_id,
|
||||
"finished_at": _now(),
|
||||
"updated_at": _now(),
|
||||
}
|
||||
if result_snapshot is not None:
|
||||
values["result_snapshot"] = result_snapshot
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(MafRoundtableSeatRunRow)
|
||||
.where(MafRoundtableSeatRunRow.id == seat_run_id)
|
||||
.values(
|
||||
status="completed",
|
||||
workflow_checkpoint_id=checkpoint_id,
|
||||
finished_at=_now(),
|
||||
updated_at=_now(),
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
@@ -163,7 +401,10 @@ class MafSeatRunRepository:
|
||||
stmt = (
|
||||
select(MafRoundtableSeatRunRow)
|
||||
.where(MafRoundtableSeatRunRow.maf_run_id == maf_run_id)
|
||||
.order_by(MafRoundtableSeatRunRow.turn_index.asc())
|
||||
.order_by(
|
||||
MafRoundtableSeatRunRow.turn_index.asc(),
|
||||
MafRoundtableSeatRunRow.execution_stage.asc(),
|
||||
)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
"""Phase 3 migration for the MAF roundtable experiment.
|
||||
|
||||
Idempotent migration that brings a Phase-2 deployment up to Phase-3 schema:
|
||||
|
||||
1. Add the non-null ``execution_stage`` column to ``maf_roundtable_seat_runs``
|
||||
(existing rows default to ``legacy``).
|
||||
2. Replace the Phase-2 three-column unique constraint
|
||||
``(maf_run_id, agent_id, turn_index)`` with the Phase-3 four-column
|
||||
constraint ``(maf_run_id, agent_id, turn_index, execution_stage)`` so the
|
||||
same business round can hold multiple distinct DeerFlow executions
|
||||
(planner / expert / format_repair / cross_review / finalizer / group_chat)
|
||||
while each stage is still unique per (run, agent, turn).
|
||||
|
||||
The business ``turn_index`` is NEVER mutated to dodge a collision — it stays
|
||||
the stable business round used by the frontend, approval matching and event
|
||||
tracing. ``execution_stage`` is the new disambiguator.
|
||||
|
||||
Dialect handling
|
||||
----------------
|
||||
- **PostgreSQL / MySQL**: ``ALTER TABLE`` to add the column, then ``DROP`` the
|
||||
old unique constraint/index and ``ADD`` the new one. Each step is guarded by
|
||||
introspection so re-runs are no-ops.
|
||||
- **SQLite**: a unique constraint cannot be dropped in place, so the table is
|
||||
rebuilt: a temp table with the new shape is created, data is copied (existing
|
||||
rows get ``execution_stage='legacy'``), the old table is dropped, and the
|
||||
temp is renamed. This is the only safe way to change a SQLite unique
|
||||
constraint; it is fully idempotent because the rebuild only runs when the
|
||||
``execution_stage`` column is absent or the old constraint is still present.
|
||||
|
||||
**Deployment order**: run AFTER the Phase-2 migration and BEFORE enabling the
|
||||
Phase-3 code path (the four-column constraint is required for the new
|
||||
``execution_stage`` writes to coexist within one turn).
|
||||
|
||||
Usage (from ``offline-backend-20260512/backend/``)::
|
||||
|
||||
# Preview (no writes)
|
||||
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase3.py --dry-run
|
||||
|
||||
# Execute
|
||||
PYTHONPATH=. uv run python scripts/migrate_maf_roundtable_phase3.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger("migrate_maf_roundtable_phase3")
|
||||
|
||||
OLD_CONSTRAINT_NAME = "uq_maf_seat_runs_run_agent_turn"
|
||||
NEW_CONSTRAINT_NAME = "uq_maf_seat_runs_run_agent_turn_stage"
|
||||
TABLE = "maf_roundtable_seat_runs"
|
||||
|
||||
|
||||
async def _migrate(dry_run: bool) -> int:
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
|
||||
|
||||
config = get_app_config()
|
||||
await init_engine_from_config(config.database)
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
logger.error("Database backend is 'memory' — nothing to migrate.")
|
||||
return 0
|
||||
|
||||
engine = sf.kw["bind"] # the async engine
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(_do_migrate_sync, dry_run=dry_run, logger=logger, changes_ref=[0])
|
||||
|
||||
await close_engine()
|
||||
logger.info("Migration complete (dry_run=%s).", dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def _do_migrate_sync(
|
||||
sync_conn: Any,
|
||||
*,
|
||||
dry_run: bool,
|
||||
logger: logging.Logger,
|
||||
changes_ref: list[int],
|
||||
) -> None:
|
||||
"""Run all migration steps on a sync connection (called via run_sync)."""
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
insp = inspect(sync_conn)
|
||||
dialect_name = sync_conn.dialect.name
|
||||
|
||||
def _exec(sql: str) -> None:
|
||||
if dry_run:
|
||||
logger.info(" [dry-run] would execute: %s", sql.strip()[:300])
|
||||
return
|
||||
sync_conn.execute(text(sql))
|
||||
changes_ref[0] += 1
|
||||
|
||||
if TABLE not in insp.get_table_names():
|
||||
logger.info("Table %s does not exist — Phase-3 migration is a no-op (table will be created by the ORM with the new schema).", TABLE)
|
||||
return
|
||||
|
||||
columns = {c["name"] for c in insp.get_columns(TABLE)}
|
||||
has_stage_col = "execution_stage" in columns
|
||||
has_snapshot_col = "result_snapshot" in columns
|
||||
|
||||
# ── 1a. Add execution_stage column (additive; old rows → 'legacy') ─────
|
||||
if not has_stage_col:
|
||||
logger.info("Adding column execution_stage to %s", TABLE)
|
||||
_exec(_ddl_add_execution_stage(dialect_name))
|
||||
else:
|
||||
logger.info("Column execution_stage already exists on %s — skip.", TABLE)
|
||||
|
||||
# ── 2. Replace the unique constraint (3-tuple → 4-tuple) ───────────────
|
||||
# NOTE: introspection is re-fetched after the column add so the inspector
|
||||
# reflects the new column within this same transaction.
|
||||
insp = inspect(sync_conn)
|
||||
|
||||
old_constraint = _find_unique_constraint(insp, TABLE, expected_cols={"maf_run_id", "agent_id", "turn_index"})
|
||||
new_constraint = _find_unique_constraint(insp, TABLE, expected_cols={"maf_run_id", "agent_id", "turn_index", "execution_stage"})
|
||||
|
||||
has_new_unique = new_constraint is not None
|
||||
has_stage_check = _has_check_constraint(insp, TABLE, "ck_maf_seat_runs_execution_stage")
|
||||
|
||||
if dialect_name == "sqlite":
|
||||
needs_rebuild = (
|
||||
not has_new_unique
|
||||
or not has_snapshot_col
|
||||
or not has_stage_check
|
||||
)
|
||||
if needs_rebuild:
|
||||
logger.info("SQLite: rebuilding %s (has_new_unique=%s, has_snapshot=%s, has_check=%s).",
|
||||
TABLE, has_new_unique, has_snapshot_col, has_stage_check)
|
||||
for stmt in _sqlite_rebuild_seat_runs(
|
||||
source_has_stage=has_stage_col,
|
||||
source_has_snapshot=has_snapshot_col,
|
||||
):
|
||||
_exec(stmt)
|
||||
else:
|
||||
logger.info("SQLite: %s already fully migrated — no-op.", TABLE)
|
||||
logger.info("Migration steps: %d applied.", changes_ref[0])
|
||||
return
|
||||
|
||||
# PostgreSQL / MySQL
|
||||
if not has_new_unique:
|
||||
if old_constraint is not None:
|
||||
old_name = old_constraint.get("name") or OLD_CONSTRAINT_NAME
|
||||
old_kind = old_constraint.get("kind")
|
||||
if old_kind == "index":
|
||||
logger.info("Dropping old unique INDEX %s (kind=index)", old_name)
|
||||
_exec(_ddl_drop_index(old_name))
|
||||
else:
|
||||
logger.info("Dropping old unique CONSTRAINT %s (kind=constraint)", old_name)
|
||||
_exec(_ddl_drop_constraint(dialect_name, old_name))
|
||||
else:
|
||||
logger.info(
|
||||
"Old unique object not found by introspection — migration assumes "
|
||||
"the four-column constraint is being created fresh."
|
||||
)
|
||||
|
||||
logger.info("Adding new four-column unique constraint %s", NEW_CONSTRAINT_NAME)
|
||||
_exec(_ddl_add_new_constraint(dialect_name))
|
||||
|
||||
if not has_snapshot_col:
|
||||
insp = inspect(sync_conn)
|
||||
cols_now = {c["name"] for c in insp.get_columns(TABLE)}
|
||||
if "result_snapshot" not in cols_now:
|
||||
logger.info("Adding column result_snapshot to %s", TABLE)
|
||||
_exec(_ddl_add_result_snapshot(dialect_name))
|
||||
|
||||
insp = inspect(sync_conn)
|
||||
if not _has_check_constraint(insp, TABLE, "ck_maf_seat_runs_execution_stage"):
|
||||
logger.info("Adding CHECK constraint ck_maf_seat_runs_execution_stage")
|
||||
_exec(_ddl_add_execution_stage_check(dialect_name))
|
||||
|
||||
logger.info("Migration steps: %d applied.", changes_ref[0])
|
||||
|
||||
|
||||
def _find_unique_constraint(insp: Any, table: str, *, expected_cols: set[str]) -> dict[str, Any] | None:
|
||||
"""Return the unique constraint (or unique index) matching ``expected_cols``.
|
||||
|
||||
Looks at both table-level unique constraints AND unique indexes — Phase-2
|
||||
created the constraint via ``CREATE UNIQUE INDEX`` which SQLAlchemy reports
|
||||
as an index, while the ORM model declares it as a UniqueConstraint (reported
|
||||
as a constraint). We accept either form so the migration is robust to how
|
||||
the table was originally created.
|
||||
"""
|
||||
for uc in insp.get_unique_constraints(table):
|
||||
if set(uc.get("column_names") or []) == expected_cols:
|
||||
return {"name": uc.get("name"), "kind": "constraint"}
|
||||
for idx in insp.get_indexes(table):
|
||||
if idx.get("unique") and set(idx.get("column_names") or []) == expected_cols:
|
||||
return {"name": idx.get("name"), "kind": "index"}
|
||||
return None
|
||||
|
||||
|
||||
def _has_check_constraint(insp: Any, table: str, name: str) -> bool:
|
||||
"""True if a CHECK constraint with the given name exists on ``table``.
|
||||
|
||||
SQLAlchemy's inspector exposes CHECK constraints via ``get_check_constraints``
|
||||
on PostgreSQL / MySQL / modern SQLite. We match by name; if the dialect
|
||||
cannot list checks we conservatively return False so the (idempotent) ADD
|
||||
runs once — a duplicate-name error there is harmless and caught by the
|
||||
``IF NOT EXISTS``-style guard in the caller's re-introspection.
|
||||
"""
|
||||
getter = getattr(insp, "get_check_constraints", None)
|
||||
if getter is None:
|
||||
return False
|
||||
try:
|
||||
for ck in getter(table):
|
||||
if ck.get("name") == name:
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# ── DDL templates (portable) ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _ddl_add_execution_stage(dialect: str) -> str:
|
||||
"""Add the execution_stage column, back-filling existing rows to 'legacy'.
|
||||
|
||||
Adding NOT NULL with a server default is safe on all three dialects: the
|
||||
default populates existing rows during the ALTER, then stays for new ones.
|
||||
"""
|
||||
if dialect == "mysql":
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
|
||||
)
|
||||
if dialect == "postgresql":
|
||||
# Postgres needs the default, and we set it server-side so existing
|
||||
# rows back-fill automatically.
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
|
||||
)
|
||||
# SQLite < 3.35 does not support adding a NOT NULL column with a default
|
||||
# via ALTER TABLE in the same way; but modern SQLite (3.35+, 2021+) does.
|
||||
# The default 'legacy' back-fills existing rows.
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy'"
|
||||
)
|
||||
|
||||
|
||||
def _ddl_add_result_snapshot(dialect: str) -> str:
|
||||
"""Add the nullable ``result_snapshot`` JSON column (Phase-3 P0-1b).
|
||||
|
||||
Stores a sanitized, replayable snapshot of the SeatResponse written
|
||||
atomically with the completed transition. Nullable so the ALTER is safe on
|
||||
all dialects (existing rows get NULL).
|
||||
"""
|
||||
if dialect == "mysql":
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN result_snapshot JSON NULL"
|
||||
)
|
||||
if dialect == "postgresql":
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN result_snapshot JSON NULL"
|
||||
)
|
||||
# SQLite — reached only when not rebuilding; use TEXT for JSON.
|
||||
return (
|
||||
"ALTER TABLE maf_roundtable_seat_runs "
|
||||
"ADD COLUMN result_snapshot TEXT"
|
||||
)
|
||||
|
||||
|
||||
def _ddl_drop_constraint(dialect: str, name: str) -> str:
|
||||
if dialect == "mysql":
|
||||
# MySQL uses DROP INDEX for both unique constraints and unique indexes.
|
||||
return f"ALTER TABLE maf_roundtable_seat_runs DROP INDEX {name}"
|
||||
# PostgreSQL / ANSI
|
||||
return f"ALTER TABLE maf_roundtable_seat_runs DROP CONSTRAINT {name}"
|
||||
|
||||
|
||||
def _ddl_drop_index(name: str) -> str:
|
||||
return f"DROP INDEX {name}"
|
||||
|
||||
|
||||
def _ddl_add_new_constraint(dialect: str) -> str:
|
||||
new = NEW_CONSTRAINT_NAME
|
||||
cols = "maf_run_id, agent_id, turn_index, execution_stage"
|
||||
if dialect == "mysql":
|
||||
return (
|
||||
f"ALTER TABLE maf_roundtable_seat_runs "
|
||||
f"ADD UNIQUE INDEX {new} ({cols})"
|
||||
)
|
||||
# PostgreSQL — table constraint
|
||||
return (
|
||||
f"ALTER TABLE maf_roundtable_seat_runs "
|
||||
f"ADD CONSTRAINT {new} UNIQUE ({cols})"
|
||||
)
|
||||
|
||||
|
||||
def _ddl_add_execution_stage_check(dialect: str) -> str:
|
||||
"""Add the closed-enum CHECK constraint on execution_stage.
|
||||
|
||||
Mirrors ``seat_runs.VALID_EXECUTION_STAGES`` so the DB itself rejects a typo
|
||||
(e.g. 'exprt') that would otherwise create a brand-new stage and bypass the
|
||||
four-column idempotency key. Keep the value list in sync with
|
||||
``VALID_EXECUTION_STAGES``.
|
||||
"""
|
||||
allowed = (
|
||||
"'legacy','planner','expert','format_repair',"
|
||||
"'cross_review','cross_review_repair','finalizer','group_chat'"
|
||||
)
|
||||
if dialect == "mysql":
|
||||
# MySQL 8 accepts inline CHECK; older MySQL silently ignores it, but the
|
||||
# app-level validation in seat_runs._validate_stage is the real guard.
|
||||
return (
|
||||
f"ALTER TABLE maf_roundtable_seat_runs "
|
||||
f"ADD CONSTRAINT ck_maf_seat_runs_execution_stage "
|
||||
f"CHECK (execution_stage IN ({allowed}))"
|
||||
)
|
||||
# PostgreSQL / ANSI
|
||||
return (
|
||||
f"ALTER TABLE maf_roundtable_seat_runs "
|
||||
f"ADD CONSTRAINT ck_maf_seat_runs_execution_stage "
|
||||
f"CHECK (execution_stage IN ({allowed}))"
|
||||
)
|
||||
|
||||
|
||||
def _sqlite_rebuild_seat_runs(
|
||||
*,
|
||||
source_has_stage: bool = False,
|
||||
source_has_snapshot: bool = False,
|
||||
) -> list[str]:
|
||||
"""Rebuild the table in SQLite to swap the unique constraint.
|
||||
|
||||
SQLite's ``ALTER TABLE`` cannot drop a constraint, so the portable recipe is:
|
||||
create a new table with the desired shape, copy data over, drop the old
|
||||
table, and rename. Indexes are recreated on the new table.
|
||||
|
||||
``source_has_stage`` / ``source_has_snapshot`` control the INSERT SELECT
|
||||
expressions so an intermediate table (already has stage and/or snapshot)
|
||||
does NOT lose data by being overwritten with defaults.
|
||||
"""
|
||||
allowed = (
|
||||
"'legacy','planner','expert','format_repair',"
|
||||
"'cross_review','cross_review_repair','finalizer','group_chat'"
|
||||
)
|
||||
source_stage = "execution_stage" if source_has_stage else "'legacy'"
|
||||
source_snapshot = "result_snapshot" if source_has_snapshot else "NULL"
|
||||
return [
|
||||
f"""
|
||||
CREATE TABLE maf_roundtable_seat_runs__phase3 (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
result_snapshot TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT {NEW_CONSTRAINT_NAME} UNIQUE (maf_run_id, agent_id, turn_index, execution_stage),
|
||||
CONSTRAINT ck_maf_seat_runs_execution_stage CHECK (execution_stage IN ({allowed}))
|
||||
)
|
||||
""",
|
||||
f"""
|
||||
INSERT INTO maf_roundtable_seat_runs__phase3 (
|
||||
id, maf_run_id, agent_id, turn_index, execution_stage,
|
||||
deerflow_run_id, deerflow_thread_id, sandbox_id, workflow_checkpoint_id,
|
||||
status, result_snapshot, started_at, finished_at, error, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
id, maf_run_id, agent_id, turn_index, {source_stage},
|
||||
deerflow_run_id, deerflow_thread_id, sandbox_id, workflow_checkpoint_id,
|
||||
status, {source_snapshot}, started_at, finished_at, error, created_at, updated_at
|
||||
FROM maf_roundtable_seat_runs
|
||||
""",
|
||||
"DROP TABLE maf_roundtable_seat_runs",
|
||||
"ALTER TABLE maf_roundtable_seat_runs__phase3 RENAME TO maf_roundtable_seat_runs",
|
||||
"CREATE INDEX ix_maf_seat_runs_maf_run_id ON maf_roundtable_seat_runs (maf_run_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_agent_id ON maf_roundtable_seat_runs (agent_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_deerflow_run_id ON maf_roundtable_seat_runs (deerflow_run_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_status ON maf_roundtable_seat_runs (status)",
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 3 migration for MAF roundtable seat runs")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview only; do not write to the database")
|
||||
args = parser.parse_args()
|
||||
return asyncio.run(_migrate(args.dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Unit tests for the Phase-3 shared fact ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.fact_ledger import (
|
||||
assert_ledger_invariants,
|
||||
empty_ledger,
|
||||
executed_action_ids,
|
||||
merge_seat_output,
|
||||
normalize_ledger,
|
||||
proposed_action_ids,
|
||||
record_action_transition,
|
||||
record_tool_result,
|
||||
)
|
||||
|
||||
|
||||
def test_empty_ledger_has_all_partitions():
|
||||
ledger = empty_ledger()
|
||||
for key in (
|
||||
"confirmed_facts",
|
||||
"evidence",
|
||||
"tool_results",
|
||||
"executed_actions",
|
||||
"pending_actions",
|
||||
"failed_or_unknown_actions",
|
||||
"conflicts",
|
||||
"proposed_actions",
|
||||
):
|
||||
assert key in ledger
|
||||
assert ledger[key] == []
|
||||
|
||||
|
||||
def test_normalize_legacy_checkpoint():
|
||||
legacy = {
|
||||
"confirmed_facts": ["a"],
|
||||
"tool_results": [{"tool_name": "t", "status": "completed"}],
|
||||
"conflicts": [],
|
||||
"pending_actions": [],
|
||||
}
|
||||
n = normalize_ledger(legacy)
|
||||
assert n["confirmed_facts"] == ["a"]
|
||||
assert n["executed_actions"] == []
|
||||
assert n["evidence"] == []
|
||||
|
||||
|
||||
def test_merge_seat_output_keeps_proposed_out_of_executed():
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(
|
||||
ledger,
|
||||
{
|
||||
"summary": "s",
|
||||
"findings": ["fact-1"],
|
||||
"evidence_refs": ["doc:1"],
|
||||
"proposed_actions": [{"text": "create ticket", "action_id": "prop-1"}],
|
||||
"executed_actions": [{"text": "forged", "action_id": "forge-1"}],
|
||||
"conflicts": ["c1"],
|
||||
},
|
||||
agent_id="seat-a",
|
||||
)
|
||||
# Phase-3: seat findings land in candidate_facts (unverified claims), NOT
|
||||
# confirmed_facts. Promotion to confirmed happens only after cross-review.
|
||||
assert ledger["confirmed_facts"] == []
|
||||
assert any(
|
||||
c.get("text") == "fact-1" for c in ledger["candidate_facts"] if isinstance(c, dict)
|
||||
)
|
||||
assert "prop-1" in proposed_action_ids(ledger)
|
||||
assert "forge-1" not in executed_action_ids(ledger)
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_record_action_transition_pending_to_executed():
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(
|
||||
ledger, action_id="a1", tool_name="create_ticket", status="pending", agent_id="s1"
|
||||
)
|
||||
assert len(ledger["pending_actions"]) == 1
|
||||
record_action_transition(
|
||||
ledger, action_id="a1", tool_name="create_ticket", status="executed", agent_id="s1"
|
||||
)
|
||||
assert ledger["pending_actions"] == []
|
||||
assert executed_action_ids(ledger) == {"a1"}
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_record_action_transition_failed_not_executed():
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="pending")
|
||||
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="unknown")
|
||||
assert executed_action_ids(ledger) == set()
|
||||
assert any(e.get("action_id") == "a2" for e in ledger["failed_or_unknown_actions"])
|
||||
|
||||
|
||||
def test_record_tool_result_status_clamp():
|
||||
ledger = empty_ledger()
|
||||
record_tool_result(ledger, tool_name="web_search", status="completed", agent_id="a")
|
||||
record_tool_result(ledger, tool_name="x", status="bogus", agent_id="a")
|
||||
assert ledger["tool_results"][0]["status"] == "completed"
|
||||
assert ledger["tool_results"][1]["status"] == "unknown"
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_assert_rejects_bad_tool_status():
|
||||
ledger = empty_ledger()
|
||||
ledger["tool_results"].append({"tool_name": "t", "status": "weird"})
|
||||
with pytest.raises(AssertionError):
|
||||
assert_ledger_invariants(ledger)
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Phase-3 acceptance tests for the fact ledger + cross-review wiring.
|
||||
|
||||
Covers P1-2 (candidate_facts separation), P1-3 (approval ledger integrity),
|
||||
and P1-1 (CrossReviewOutput validation + deterministic degradation).
|
||||
|
||||
These are pure-unit tests over the fact_ledger / seat_output / workflow_executors
|
||||
helpers — no DB, no real DeerFlow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.fact_ledger import (
|
||||
candidate_id_map,
|
||||
confirm_candidate_facts,
|
||||
empty_ledger,
|
||||
merge_seat_output,
|
||||
record_action_transition,
|
||||
)
|
||||
from app.maf_roundtable.seat_output import parse_cross_review_output
|
||||
|
||||
# ── P1-2: seat findings → candidate_facts, NOT confirmed_facts ─────────────
|
||||
|
||||
|
||||
def test_seat_findings_land_in_candidate_facts_not_confirmed():
|
||||
"""A seat's `findings` must NEVER be auto-promoted to confirmed_facts —
|
||||
weak model output is a claim, not a fact."""
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["市场增长10%", "竞品A领先"]}, agent_id="seat-1")
|
||||
|
||||
assert len(ledger["candidate_facts"]) == 2
|
||||
assert ledger["confirmed_facts"] == [], "seat findings must not be confirmed facts"
|
||||
|
||||
# Each candidate gets a server-generated candidate_id.
|
||||
for c in ledger["candidate_facts"]:
|
||||
assert "candidate_id" in c and c["candidate_id"]
|
||||
assert c["confirmed"] is False
|
||||
assert c["agent_id"] == "seat-1"
|
||||
assert c["source"] == "seat_finding"
|
||||
|
||||
|
||||
def test_candidate_facts_promoted_only_after_cross_review_confirmation():
|
||||
"""Promotion to confirmed_facts happens only via confirm_candidate_facts,
|
||||
and only for ids that actually exist in candidate_facts AND cite evidence
|
||||
the ledger has backing for."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A", "事实B"]}, agent_id="seat-1")
|
||||
ids = [c["candidate_id"] for c in ledger["candidate_facts"]]
|
||||
assert len(ids) == 2
|
||||
|
||||
# Record real evidence so the confirmation can cite it (Phase-3 P1-1).
|
||||
record_tool_result(
|
||||
ledger, tool_name="web_search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
|
||||
# Model tries to confirm one real id + one HALLUCINATED id, citing the
|
||||
# real evidence for both.
|
||||
tool_ref = "tool:web_search:1"
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[ids[0], "nonexistent-id"],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=[tool_ref],
|
||||
)
|
||||
|
||||
assert len(ledger["confirmed_facts"]) == 1 # only the real one
|
||||
assert ledger["confirmed_facts"][0]["candidate_id"] == ids[0]
|
||||
assert ledger["confirmed_facts"][0]["confirmed_by"] == "manager-1"
|
||||
assert ledger["confirmed_facts"][0]["source"] == "cross_review_confirmed"
|
||||
# The confirmed candidate is marked confirmed=True in place.
|
||||
assert candidate_id_map(ledger)[ids[0]]["confirmed"] is True
|
||||
assert candidate_id_map(ledger)[ids[1]]["confirmed"] is False
|
||||
|
||||
|
||||
def test_candidate_not_promoted_without_ledger_backed_evidence():
|
||||
"""Phase-3 P1-1: confirmation requires evidence that resolves to a ledger
|
||||
entry. A candidate cited with an invented evidence ref (not recorded in the
|
||||
ledger) is REJECTED and stays a candidate."""
|
||||
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
# No evidence recorded in the ledger; the model cites an invented ref.
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[cid],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=["invented-evidence-not-in-ledger"],
|
||||
)
|
||||
|
||||
assert ledger["confirmed_facts"] == [], "candidate promoted with non-ledger evidence"
|
||||
assert candidate_id_map(ledger)[cid]["confirmed"] is False
|
||||
|
||||
|
||||
def test_candidate_not_promoted_with_no_evidence_at_all():
|
||||
"""Phase-3 P1-1: a confirmation that supplies no evidence_refs is rejected."""
|
||||
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[cid],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=None,
|
||||
)
|
||||
|
||||
assert ledger["confirmed_facts"] == []
|
||||
assert candidate_id_map(ledger)[cid]["confirmed"] is False
|
||||
|
||||
|
||||
def test_confirm_is_idempotent():
|
||||
"""Confirming the same candidate_id twice does not duplicate it."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
ref = "tool:search:1"
|
||||
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
|
||||
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
|
||||
assert len(ledger["confirmed_facts"]) == 1
|
||||
|
||||
|
||||
def test_empty_ledger_has_candidate_partition():
|
||||
ledger = empty_ledger()
|
||||
assert "candidate_facts" in ledger and ledger["candidate_facts"] == []
|
||||
assert "confirmed_facts" in ledger
|
||||
|
||||
|
||||
# ── P1-3: approval ledger integrity ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pending_action_transitions_on_reject():
|
||||
"""A pending action that is rejected moves to the failed/unknown partition
|
||||
with status 'rejected' — it must not stay in pending and must not appear in
|
||||
executed."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
|
||||
status="pending", agent_id="seat-1")
|
||||
assert len(ledger["pending_actions"]) == 1
|
||||
|
||||
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
|
||||
status="rejected", agent_id="seat-1")
|
||||
|
||||
# Removed from pending.
|
||||
assert not any(p.get("action_id") == "act-1" for p in ledger["pending_actions"])
|
||||
# Present in failed/unknown with status rejected.
|
||||
failed = [f for f in ledger["failed_or_unknown_actions"] if f.get("action_id") == "act-1"]
|
||||
assert len(failed) == 1
|
||||
assert failed[0]["status"] == "rejected"
|
||||
# NOT in executed.
|
||||
assert not any(e.get("action_id") == "act-1" for e in ledger["executed_actions"])
|
||||
|
||||
|
||||
def test_pending_action_transitions_on_execute():
|
||||
"""A pending action that executes successfully moves pending → executed."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
|
||||
status="pending", agent_id="seat-1")
|
||||
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
|
||||
status="executed", agent_id="seat-1")
|
||||
assert not any(p.get("action_id") == "act-2" for p in ledger["pending_actions"])
|
||||
assert any(e.get("action_id") == "act-2" and e.get("status") == "executed"
|
||||
for e in ledger["executed_actions"])
|
||||
assert not any(f.get("action_id") == "act-2" for f in ledger["failed_or_unknown_actions"])
|
||||
|
||||
|
||||
def test_failed_action_never_in_executed():
|
||||
"""A failed/unknown action must never leak into executed_actions."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
|
||||
status="pending", agent_id="seat-1")
|
||||
for terminal in ("failed", "unknown"):
|
||||
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
|
||||
status=terminal, agent_id="seat-1")
|
||||
assert not any(e.get("action_id") == "act-3" for e in ledger["executed_actions"]), \
|
||||
f"{terminal} action leaked into executed_actions"
|
||||
|
||||
|
||||
# ── P1-1: CrossReviewOutput parsing ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_cross_review_output_valid():
|
||||
payload = {
|
||||
"summary": "审查通过",
|
||||
"evidence_ok": True,
|
||||
"gaps": ["缺少 X 证据"],
|
||||
"conflicts": [],
|
||||
"tool_failures": [],
|
||||
"write_status_notes": ["待审批1个"],
|
||||
"recommendations": ["补充X"],
|
||||
"confirmed_candidate_ids": ["c1", "c2"],
|
||||
}
|
||||
model, errors = parse_cross_review_output(json.dumps(payload))
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
assert model.summary == "审查通过"
|
||||
assert model.confirmed_candidate_ids == ["c1", "c2"]
|
||||
|
||||
|
||||
def test_parse_cross_review_confirmed_candidates_requires_evidence():
|
||||
"""Phase-3 P1-1: each ConfirmedCandidate must carry ≥1 evidence ref — an
|
||||
empty list is a schema violation (so the executor repairs instead of
|
||||
silently accepting a bare-id confirmation)."""
|
||||
payload = {
|
||||
"summary": "ok",
|
||||
"confirmed_candidates": [{"candidate_id": "c1", "evidence_refs": []}],
|
||||
}
|
||||
model, errors = parse_cross_review_output(json.dumps(payload))
|
||||
assert model is None
|
||||
assert any("evidence_refs" in e or "confirmed_candidates" in e for e in errors)
|
||||
|
||||
|
||||
def test_parse_cross_review_evidence_ok_defaults_false():
|
||||
"""Phase-3 P1-1: omitting evidence_ok must default to False — a model that
|
||||
forgets the field must never be treated as 'evidence sufficient'."""
|
||||
model, errors = parse_cross_review_output(json.dumps({"summary": "ok"}))
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
assert model.evidence_ok is False
|
||||
|
||||
|
||||
def test_parse_cross_review_output_invalid_json_returns_errors():
|
||||
model, errors = parse_cross_review_output("not json at all")
|
||||
assert model is None
|
||||
assert errors # non-empty readable errors
|
||||
|
||||
|
||||
def test_parse_cross_review_output_missing_summary_invalid():
|
||||
"""summary is required (min_length=1) — a missing summary must fail
|
||||
validation so the CrossReviewExecutor repairs rather than accepting it."""
|
||||
model, errors = parse_cross_review_output(json.dumps({"gaps": ["x"]}))
|
||||
assert model is None
|
||||
assert any("summary" in e for e in errors)
|
||||
|
||||
|
||||
# ── P1-1: CrossReviewExecutor deterministic degradation ────────────────────
|
||||
#
|
||||
# These exercise the executor's _apply_cross_review_output path with a fake
|
||||
# adapter, proving: (a) valid output confirms candidates; (b) invalid output
|
||||
# that the repair also cannot fix degrades deterministically WITHOUT pasting
|
||||
# raw JSON into summary.
|
||||
|
||||
|
||||
def _make_state():
|
||||
"""Minimal RoundtableState for CrossReviewExecutor tests."""
|
||||
from app.maf_roundtable.workflow_executors import RoundtableState
|
||||
|
||||
return RoundtableState(
|
||||
session_id="s1", run_id="r1", task_text="t", mode="structured",
|
||||
model_name=None, reasoning_effort=None, max_rounds=8,
|
||||
write_policy="confirm_high_risk", manager_agent_id="mgr",
|
||||
seat_agent_ids=["seat-1"], thread_map={"mgr": "t-mgr"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_valid_output_confirms_candidates():
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
# Record evidence so the confirmation can bind it (Phase-3 P1-1).
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
# Structured confirmed_candidates path: each candidate cites ledger evidence.
|
||||
raw = json.dumps({
|
||||
"summary": "ok",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert review["source"] == "manager"
|
||||
assert review["schema_status"] == "ok"
|
||||
assert review["summary"] == "ok"
|
||||
# The candidate was promoted to confirmed_facts.
|
||||
assert len(state.fact_ledger["confirmed_facts"]) == 1
|
||||
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
|
||||
# The public confirmation field contains only the server-accepted item;
|
||||
# the model request is retained separately for audit.
|
||||
assert review["confirmed_candidate_ids"] == [candidate_id]
|
||||
assert review["confirmed_candidates"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
|
||||
]
|
||||
assert review["requested_confirmations"] == review["confirmed_candidates"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_rejects_candidate_with_invented_evidence():
|
||||
"""Phase-3 P1-1: a confirmation that cites evidence NOT in the ledger is
|
||||
rejected server-side. The candidate stays unconfirmed, and the rejection is
|
||||
surfaced in the review so the summary can explain it."""
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "ok",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
# Not promoted.
|
||||
assert state.fact_ledger["confirmed_facts"] == []
|
||||
# Rejection recorded.
|
||||
assert review.get("confirmation_rejections"), "rejection not surfaced in review"
|
||||
rej = review["confirmation_rejections"][0]
|
||||
assert rej["candidate_id"] == candidate_id
|
||||
assert rej["reason"] == "evidence_not_in_ledger"
|
||||
assert review["confirmed_candidates"] == []
|
||||
assert review["confirmed_candidate_ids"] == []
|
||||
assert review["requested_confirmations"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_invalid_then_repair_succeeds():
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
# Fake adapter: repair turn returns valid JSON.
|
||||
good_repair = json.dumps({"summary": "repaired ok", "confirmed_candidate_ids": []})
|
||||
fake_adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content=good_repair)),
|
||||
)
|
||||
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
|
||||
review: dict = {}
|
||||
# First output is garbage; repair returns valid.
|
||||
await executor._apply_cross_review_output(review, "totally not json", "mgr")
|
||||
|
||||
assert review["source"] == "manager"
|
||||
assert review["schema_status"] == "repaired"
|
||||
assert review["summary"] == "repaired ok"
|
||||
fake_adapter.run_seat_turn.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_invalid_and_repair_fails_degrades_deterministically():
|
||||
"""When both the original and the repair output are unparseable, the
|
||||
review MUST degrade deterministically: raw JSON is never pasted into
|
||||
summary, and the review is marked schema_invalid."""
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
# Repair also returns garbage.
|
||||
fake_adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content="still garbage")),
|
||||
)
|
||||
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
|
||||
review: dict = {}
|
||||
await executor._apply_cross_review_output(review, "garbage first output", "mgr")
|
||||
|
||||
assert review["source"] == "deterministic"
|
||||
assert review["schema_status"] == "schema_invalid"
|
||||
assert review.get("raw_response_ignored") is True
|
||||
# summary must NOT be the raw model JSON.
|
||||
assert review.get("summary", "") != "garbage first output"
|
||||
|
||||
|
||||
# ── P1-2: evidence_ok=False blocks all promotion ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_ok_false_blocks_promotion_even_with_valid_refs():
|
||||
"""Phase-3 P1-2: when the model says evidence_ok=False, no candidate may
|
||||
be promoted even if it cites valid ledger-backed evidence refs. The
|
||||
attempted confirmations are recorded as rejections."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "证据不充分",
|
||||
"evidence_ok": False,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert state.fact_ledger["confirmed_facts"] == []
|
||||
assert candidate_id_map(state.fact_ledger)[candidate_id]["confirmed"] is False
|
||||
assert review.get("confirmation_rejections")
|
||||
rej = review["confirmation_rejections"][0]
|
||||
assert rej["candidate_id"] == candidate_id
|
||||
assert rej["reason"] == "review_did_not_confirm_evidence"
|
||||
assert review["confirmed_candidates"] == []
|
||||
assert review["confirmed_candidate_ids"] == []
|
||||
assert review["requested_confirmations"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_ok_true_allows_promotion_with_valid_refs():
|
||||
"""Phase-3 P1-2: when evidence_ok=True AND evidence refs are valid, the
|
||||
candidate IS promoted."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实B"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "证据充分",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert len(state.fact_ledger["confirmed_facts"]) == 1
|
||||
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
|
||||
@@ -0,0 +1,619 @@
|
||||
"""Phase-3 Round-2 acceptance tests (P0-1): race-free idempotency & replay.
|
||||
|
||||
These prove the Phase-3 promise: a distinct DeerFlow execution (run, agent,
|
||||
turn, execution_stage) is started at most once, even under concurrency,
|
||||
retries, recovery, or a partial crash. They use a REAL SQLite-backed store +
|
||||
the REAL ``DeerFlowAgentAdapter`` (only ``_execute_run`` is faked, so the
|
||||
adapter's get_or_create / status-branching / snapshot logic is exercised
|
||||
end-to-end against a real DB).
|
||||
|
||||
Coverage:
|
||||
- Two coroutines acquiring the same four-tuple concurrently → exactly one wins
|
||||
(created=True), the other reads the existing row (created=False). This is the
|
||||
race that "query-then-insert" lost; the atomic insert-then-read wins it.
|
||||
- A double-call of ``run_seat_turn`` after completion does NOT start a second
|
||||
DeerFlow run — it replays the saved ``result_snapshot``.
|
||||
- A call while the stage is pending/running returns an in-flight marker and
|
||||
does NOT start a second DeerFlow run.
|
||||
- A failed/cancelled stage is terminal and never auto-retried.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter, SeatResponse
|
||||
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def svc(tmp_path):
|
||||
# IMPORTANT: a file-based SQLite is required for the concurrency test. The
|
||||
# in-memory engine (``sqlite+aiosqlite://``) multiplexes all sessions over a
|
||||
# SINGLE connection, so two coroutines in ``asyncio.gather`` share one
|
||||
# transaction — the loser's rollback would void the winner's uncommitted
|
||||
# insert, making a true race impossible to simulate. A file DB gives each
|
||||
# ``async with self._sf()`` its own connection (production behavior), so the
|
||||
# four-column unique constraint is the real arbiter.
|
||||
db_path = tmp_path / "idempotency.db"
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
service = make_maf_service(store)
|
||||
|
||||
# Seed a session + run so the seat_runs FK is satisfiable.
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="sess-1", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="running", write_policy="auto",
|
||||
max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="run-1", session_id="sess-1", status="running",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
yield service
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _adapter(svc) -> DeerFlowAgentAdapter:
|
||||
app = MagicMock()
|
||||
app.state.checkpointer = AsyncMock()
|
||||
app.state.thread_store = AsyncMock()
|
||||
app.state.stream_bridge = None
|
||||
app.state.run_manager = AsyncMock()
|
||||
client = DeerFlowRuntimeClient(app)
|
||||
return DeerFlowAgentAdapter(client, service=svc, user_id="u1")
|
||||
|
||||
|
||||
# ── 1. Concurrent acquisition: exactly one wins ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_get_or_create_one_winner(svc):
|
||||
"""Two coroutines acquire the same four-tuple at (near-)the same time. The
|
||||
atomic insert-then-read means the DB constraint picks a single winner; the
|
||||
loser reads the winner's row. Both calls return, neither raises, and there
|
||||
is exactly ONE row for the four-tuple."""
|
||||
repo = svc.store.seat_runs
|
||||
key = dict(maf_run_id="run-1", agent_id="a1", turn_index=1, execution_stage="expert")
|
||||
|
||||
(ar, ac), (br, bc) = await asyncio.gather(
|
||||
repo.get_or_create_seat_run(**key),
|
||||
repo.get_or_create_seat_run(**key),
|
||||
)
|
||||
assert (ac, bc) in {(True, False), (False, True)}, "exactly one winner; the other reads existing"
|
||||
assert ar["id"] == br["id"]
|
||||
rows = await repo.get_by_maf_run("run-1")
|
||||
assert len([r for r in rows if r["agent_id"] == "a1" and r["execution_stage"] == "expert"]) == 1
|
||||
|
||||
|
||||
# ── 2. Double-call after completion replays the snapshot ────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_double_call_after_completion_does_not_re_run(svc):
|
||||
"""A second ``run_seat_turn`` for an already-completed stage must replay the
|
||||
saved ``result_snapshot`` and NOT call ``_execute_run`` again. This is the
|
||||
core recovery guarantee: a restart or retry never re-invokes the model."""
|
||||
adapter = _adapter(svc)
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"],
|
||||
turn_index=kwargs["turn_index"],
|
||||
content="first-run-output",
|
||||
tool_calls=[{"name": "search", "args": {}, "id": "tc1"}],
|
||||
usage={"input_tokens": 5, "output_tokens": 5},
|
||||
metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a2", turn_index=2,
|
||||
execution_stage="expert", message="q", thread_id="t-a2",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1
|
||||
assert first.content == "first-run-output"
|
||||
|
||||
# Second call — completed stage. Must replay, NOT re-run.
|
||||
second = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1, "completed stage was re-run instead of replayed"
|
||||
assert second.content == "first-run-output"
|
||||
assert second.error is None
|
||||
|
||||
|
||||
# ── 3. In-flight stage: second caller waits and replays ─────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_running_stage_second_caller_waits_and_replays(svc):
|
||||
"""When a stage is pending/running (another caller owns it), a second
|
||||
``run_seat_turn`` must WAIT for the first to complete, then replay the
|
||||
same snapshot. ``_execute_run`` is called exactly once."""
|
||||
adapter = _adapter(svc)
|
||||
execute_started = asyncio.Event()
|
||||
release_execute = asyncio.Event()
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _blocking_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
execute_started.set()
|
||||
await release_execute.wait()
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="done-content", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
|
||||
usage={"input_tokens": 1}, metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a3", turn_index=3,
|
||||
execution_stage="cross_review", message="q", thread_id="t-a3",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first_task = asyncio.create_task(adapter.run_seat_turn(**common))
|
||||
await execute_started.wait()
|
||||
|
||||
second_task = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
release_execute.set()
|
||||
first = await first_task
|
||||
second = await second_task
|
||||
|
||||
assert calls["execute"] == 1, "in-flight stage started a second DeerFlow run"
|
||||
assert first.content == "done-content"
|
||||
assert second.content == "done-content"
|
||||
assert second.error is None
|
||||
|
||||
|
||||
# ── 4. Failed/cancelled stage is terminal ────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_stage_not_auto_retried(svc):
|
||||
"""A previously-failed stage is terminal; a new ``run_seat_turn`` surfaces
|
||||
a terminal marker and never auto-retries (no silent re-run)."""
|
||||
adapter = _adapter(svc)
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _failing_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
raise RuntimeError("model blew up")
|
||||
|
||||
adapter._execute_run = _failing_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a4", turn_index=4,
|
||||
execution_stage="finalizer", message="q", thread_id="t-a4",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first = await adapter.run_seat_turn(**common)
|
||||
assert first.error # failed
|
||||
assert calls["execute"] == 1
|
||||
|
||||
second = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1, "failed stage was auto-retried"
|
||||
assert second.error is not None
|
||||
assert second.metadata is not None and second.metadata.get("terminal") == "failed"
|
||||
|
||||
|
||||
# ── 5. Four-tuple replay snapshot persists sanitized content ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_result_snapshot_persisted_on_completion(svc):
|
||||
"""The sanitized result snapshot is written atomically with the completed
|
||||
transition so recovery can replay it even if the workflow checkpoint was
|
||||
not yet written."""
|
||||
adapter = _adapter(svc)
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="snapshot-me", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
|
||||
usage={"input_tokens": 1}, metadata={"thread_id": "t"},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a5", turn_index=5,
|
||||
execution_stage="expert", message="q", thread_id="t-a5",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
|
||||
row = await svc.store.seat_runs.get_by_four_tuple(
|
||||
maf_run_id="run-1", agent_id="a5", turn_index=5, execution_stage="expert",
|
||||
)
|
||||
assert row is not None
|
||||
assert row["status"] == "completed"
|
||||
snap = row["result_snapshot"]
|
||||
assert snap is not None
|
||||
assert snap["content"] == "snapshot-me"
|
||||
assert snap["tool_calls"][0]["name"] == "t"
|
||||
|
||||
|
||||
# ── 6. Invalid execution_stage rejected ──────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_execution_stage_rejected(svc):
|
||||
"""A typo'd execution_stage (e.g. 'exprt') is rejected at the repository —
|
||||
it would otherwise bypass the four-column idempotency key by creating a
|
||||
brand-new stage."""
|
||||
from deerflow.persistence.maf_roundtable.seat_runs import InvalidExecutionStageError
|
||||
|
||||
repo = svc.store.seat_runs
|
||||
with pytest.raises(InvalidExecutionStageError):
|
||||
await repo.get_or_create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a6", turn_index=6,
|
||||
execution_stage="exprt", # typo
|
||||
)
|
||||
with pytest.raises(InvalidExecutionStageError):
|
||||
await repo.create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a6", turn_index=6,
|
||||
execution_stage="expert2", # not in the closed enum
|
||||
)
|
||||
|
||||
|
||||
# ── 7. Migration branching: constraint vs index drop ─────────────────────────
|
||||
|
||||
|
||||
def _load_phase3_migration():
|
||||
here = Path(__file__).resolve().parent.parent
|
||||
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
|
||||
assert spec and spec.loader, "could not load phase3 migration script"
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def test_migration_ddl_drop_branches_on_kind():
|
||||
"""The migration must emit DROP CONSTRAINT for a table constraint and DROP
|
||||
INDEX for a unique index — the Phase-2 deployment created the constraint as
|
||||
a CREATE UNIQUE INDEX, so calling DROP CONSTRAINT on it failed."""
|
||||
mod = _load_phase3_migration()
|
||||
# A constraint-kind object → DROP CONSTRAINT.
|
||||
drop_constraint_sql = mod._ddl_drop_constraint("postgresql", "uq_maf_seat_runs_run_agent_turn")
|
||||
assert "DROP CONSTRAINT" in drop_constraint_sql
|
||||
# An index-kind object → DROP INDEX.
|
||||
drop_index_sql = mod._ddl_drop_index("uq_maf_seat_runs_run_agent_turn")
|
||||
assert "DROP INDEX" in drop_index_sql
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_sqlite_rebuild_restores_deerflow_run_id_index():
|
||||
"""The SQLite rebuild must recreate the ``deerflow_run_id`` index — without
|
||||
it, cancel-by-deerflow-run-id lookups silently degrade after a restart."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
|
||||
joined = "\n".join(rebuild)
|
||||
assert "deerflow_run_id" in joined, "rebuild dropped the deerflow_run_id index"
|
||||
assert "ck_maf_seat_runs_execution_stage" in joined
|
||||
assert "execution_stage IN" in joined
|
||||
|
||||
|
||||
# ── 8. P0-1: Two adapters same stage — wait, not partial_failure ─────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_adapters_same_stage_wait_not_partial_failure(svc):
|
||||
"""Two adapter calls for the same four-tuple: the first blocks executing,
|
||||
the second waits. After release, both get the same snapshot. _execute_run
|
||||
is called exactly once. Neither sets partial_failure."""
|
||||
adapter = _adapter(svc)
|
||||
execute_started = asyncio.Event()
|
||||
release_execute = asyncio.Event()
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _blocking_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
execute_started.set()
|
||||
await release_execute.wait()
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="shared-result", tool_calls=[], usage={"input_tokens": 2},
|
||||
metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-wait", turn_index=7,
|
||||
execution_stage="expert", message="q", thread_id="t-wait",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
task1 = asyncio.create_task(adapter.run_seat_turn(**common))
|
||||
await execute_started.wait()
|
||||
|
||||
task2 = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
release_execute.set()
|
||||
r1, r2 = await asyncio.gather(task1, task2)
|
||||
|
||||
assert calls["execute"] == 1
|
||||
assert r1.content == "shared-result"
|
||||
assert r2.content == "shared-result"
|
||||
assert r1.error is None
|
||||
assert r2.error is None
|
||||
|
||||
|
||||
# ── 9. P0-1: Empty snapshot defense ──────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_with_empty_snapshot_returns_error(svc):
|
||||
"""A completed seat run with no result_snapshot must NOT produce a fake
|
||||
success — it returns an explicit error requiring manual recovery."""
|
||||
adapter = _adapter(svc)
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
seat_run = await svc.store.seat_runs.create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a-empty", turn_index=8, execution_stage="expert",
|
||||
)
|
||||
await svc.store.seat_runs.mark_completed(seat_run["id"], result_snapshot=None)
|
||||
|
||||
result = await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-empty", turn_index=8,
|
||||
execution_stage="expert", message="q", thread_id="t-empty",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
assert result.error is not None
|
||||
assert "no replayable snapshot" in result.error
|
||||
assert result.content == ""
|
||||
|
||||
|
||||
# ── 10. P0-2: Snapshot sanitization before persist ───────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_result_snapshot_sanitized_before_persist(svc):
|
||||
"""Secrets in tool_calls args and metadata must be redacted in the
|
||||
persisted snapshot. Safe fields are preserved."""
|
||||
adapter = _adapter(svc)
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="analysis complete",
|
||||
tool_calls=[{
|
||||
"name": "request_api",
|
||||
"args": {
|
||||
"api_key": "secret-api-key",
|
||||
"password": "secret-password",
|
||||
"normal": "safe",
|
||||
},
|
||||
"id": "tc1",
|
||||
}],
|
||||
usage={"input_tokens": 1},
|
||||
metadata={"authorization": "Bearer secret-token", "thread_id": "t"},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
|
||||
execution_stage="expert", message="q", thread_id="t-san",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
|
||||
row = await svc.store.seat_runs.get_by_four_tuple(
|
||||
maf_run_id="run-1", agent_id="a-san", turn_index=9, execution_stage="expert",
|
||||
)
|
||||
assert row is not None
|
||||
snap = row["result_snapshot"]
|
||||
assert snap is not None
|
||||
snap_str = str(snap)
|
||||
assert "secret-api-key" not in snap_str
|
||||
assert "secret-password" not in snap_str
|
||||
assert "secret-token" not in snap_str
|
||||
assert snap["tool_calls"][0]["args"]["normal"] == "safe"
|
||||
|
||||
replayed = await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
|
||||
execution_stage="expert", message="q", thread_id="t-san",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
assert replayed.content == "analysis complete"
|
||||
assert "secret-api-key" not in str(replayed.tool_calls)
|
||||
|
||||
|
||||
# ── 11. P1-1: Migration intermediate state tests ─────────────────────────────
|
||||
|
||||
|
||||
def test_migration_rebuild_preserves_existing_stage_and_snapshot():
|
||||
"""When the source table already has execution_stage and result_snapshot,
|
||||
the rebuild must copy them verbatim — not overwrite with 'legacy'/NULL."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=True, source_has_snapshot=True)
|
||||
insert_sql = rebuild[1]
|
||||
assert "execution_stage" in insert_sql
|
||||
assert "'legacy'" not in insert_sql.split("SELECT")[1]
|
||||
assert "result_snapshot" in insert_sql
|
||||
assert "NULL" not in insert_sql.split("SELECT")[1].split("FROM")[0].replace("result_snapshot", "")
|
||||
|
||||
|
||||
def test_migration_rebuild_from_phase2_uses_defaults():
|
||||
"""When the source table has neither stage nor snapshot (Phase-2), the
|
||||
rebuild uses 'legacy' and NULL."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
|
||||
insert_sql = rebuild[1]
|
||||
select_part = insert_sql.split("SELECT")[1].split("FROM")[0]
|
||||
assert "'legacy'" in select_part
|
||||
assert "NULL" in select_part
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_idempotent_double_run(tmp_path):
|
||||
"""Running the migration twice on the same SQLite DB succeeds both times."""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "double_migrate.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_maf_seat_runs_run_agent_turn UNIQUE (maf_run_id, agent_id, turn_index)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
INSERT INTO maf_roundtable_seat_runs
|
||||
(id, maf_run_id, agent_id, turn_index, status, created_at, updated_at)
|
||||
VALUES ('sr1', 'run-1', 'a1', 1, 'completed', '2026-01-01', '2026-01-01')
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
mod = _load_phase3_migration()
|
||||
|
||||
for _ in range(2):
|
||||
with engine.connect() as sync_conn:
|
||||
mod._do_migrate_sync(
|
||||
sync_conn, dry_run=False, logger=logger, changes_ref=[0]
|
||||
)
|
||||
sync_conn.commit()
|
||||
|
||||
insp = inspect(engine)
|
||||
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
|
||||
assert "execution_stage" in cols
|
||||
assert "result_snapshot" in cols
|
||||
|
||||
with engine.connect() as conn:
|
||||
row = conn.execute(text("SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr1'")).fetchone()
|
||||
assert row[0] == "legacy"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_intermediate_preserves_expert_stage(tmp_path):
|
||||
"""An intermediate table (has stage + 4-col unique, but no snapshot/check)
|
||||
must preserve existing execution_stage='expert' and result_snapshot data
|
||||
after rebuild."""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "intermediate.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
result_snapshot TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_maf_seat_runs_run_agent_turn_stage UNIQUE (maf_run_id, agent_id, turn_index, execution_stage)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
INSERT INTO maf_roundtable_seat_runs
|
||||
(id, maf_run_id, agent_id, turn_index, execution_stage, status, result_snapshot, created_at, updated_at)
|
||||
VALUES ('sr2', 'run-1', 'a1', 1, 'expert', 'completed', '{"content": "keep-me"}', '2026-01-01', '2026-01-01')
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
mod = _load_phase3_migration()
|
||||
|
||||
with engine.connect() as sync_conn:
|
||||
mod._do_migrate_sync(sync_conn, dry_run=False, logger=logger, changes_ref=[0])
|
||||
sync_conn.commit()
|
||||
|
||||
with engine.connect() as conn:
|
||||
row = conn.execute(text(
|
||||
"SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr2'"
|
||||
)).fetchone()
|
||||
assert row[0] == "expert", f"stage corrupted to {row[0]!r}"
|
||||
assert "keep-me" in (row[1] or ""), f"snapshot corrupted to {row[1]!r}"
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Phase-3 Round-2 acceptance tests (P0-3): real manager workflow coexistence.
|
||||
|
||||
Drives the REAL workflow executors (PlannerExecutor → SeatExecutor ×N →
|
||||
CrossReviewExecutor → ApprovalGateExecutor → FinalizerExecutor) through a
|
||||
manager + 2-seat structured run, with a fake DeerFlow adapter whose
|
||||
``run_seat_turn`` records each call's ``execution_stage`` into a REAL seat-run
|
||||
store. This proves the Phase-3 disambiguation invariants end-to-end:
|
||||
|
||||
- manager planner / cross_review / finalizer all share the SAME business round
|
||||
(turn_index) but occupy distinct seat-run rows via ``execution_stage`` — they
|
||||
must coexist without a four-tuple collision.
|
||||
- an expert seat turn and its one-shot ``format_repair`` retry coexist for the
|
||||
same (agent, turn).
|
||||
- invalid expert output triggers exactly one ``format_repair`` DeerFlow call,
|
||||
and both rows persist.
|
||||
- invalid cross-review output triggers exactly one ``cross_review_repair``
|
||||
DeerFlow call, and ``cross_review`` + ``cross_review_repair`` rows coexist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.deerflow_agent_adapter import SeatResponse
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
)
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
|
||||
|
||||
MANAGER = "manager-1"
|
||||
SEATS = ["seat-a", "seat-b"]
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[object] = []
|
||||
self.outputs: list[object] = []
|
||||
|
||||
async def send_message(self, msg: object) -> None:
|
||||
self.messages.append(msg)
|
||||
|
||||
async def yield_output(self, msg: object) -> None:
|
||||
self.outputs.append(msg)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def svc():
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
service = make_maf_service(store)
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="sess-1", user_id="u1", title="t", task_text="分析市场",
|
||||
mode="structured", status="running", write_policy="auto",
|
||||
manager_agent_id=MANAGER, max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="run-1", session_id="sess-1", status="running",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
yield service
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _state() -> RoundtableState:
|
||||
return RoundtableState(
|
||||
session_id="sess-1", run_id="run-1", task_text="分析市场",
|
||||
mode="structured", model_name=None, reasoning_effort=None,
|
||||
max_rounds=8, write_policy="auto", manager_agent_id=MANAGER,
|
||||
seat_agent_ids=SEATS,
|
||||
thread_map={MANAGER: "t-mgr", "seat-a": "t-a", "seat-b": "t-b"},
|
||||
capability_snapshot=None, seat_timeout_seconds=5, schema_repair_enabled=True,
|
||||
)
|
||||
|
||||
|
||||
def _resp(content: str) -> SeatResponse:
|
||||
return SeatResponse(
|
||||
agent_id="x", turn_index=0, content=content,
|
||||
tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1},
|
||||
)
|
||||
|
||||
|
||||
class _RecordingAdapter:
|
||||
"""Fake adapter that (a) records every run_seat_turn's execution_stage and
|
||||
(b) writes a REAL seat-run row for each call so the four-tuple coexistence
|
||||
invariant is checked against the actual DB."""
|
||||
|
||||
def __init__(self, service) -> None:
|
||||
self._service = service
|
||||
self.calls: list[tuple[str, str, int, str]] = [] # (agent, turn, stage)
|
||||
self._seat_outputs: dict[tuple[str, str], str] = {} # (agent, stage) -> content
|
||||
self._user_id = "u1"
|
||||
self._emit_critical = AsyncMock()
|
||||
self._emit = AsyncMock()
|
||||
|
||||
def set_output(self, agent_id: str, execution_stage: str, content: str) -> None:
|
||||
self._seat_outputs[(agent_id, execution_stage)] = content
|
||||
|
||||
async def run_seat_turn(self, **kwargs):
|
||||
agent_id = kwargs["agent_id"]
|
||||
turn_index = kwargs["turn_index"]
|
||||
stage = kwargs.get("execution_stage", "expert")
|
||||
self.calls.append((agent_id, turn_index, stage))
|
||||
# Persist a seat-run row so coexistence is verified against the real DB.
|
||||
await self._service.store.seat_runs.create_seat_run(
|
||||
maf_run_id=kwargs["run_id"], agent_id=agent_id,
|
||||
turn_index=turn_index, execution_stage=stage,
|
||||
)
|
||||
content = self._seat_outputs.get((agent_id, stage), '{"summary":"ok"}')
|
||||
return _resp(content)
|
||||
|
||||
|
||||
# ── 1. Full structured run: manager stages coexist ───────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_structured_run_manager_stages_coexist(svc):
|
||||
"""A complete planner→experts→cross_review→approval→finalizer run must
|
||||
persist manager planner/cross_review/finalizer rows that ALL share the same
|
||||
business round (turn_index) but differ by execution_stage — no collision."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
# Manager emits valid JSON at each stage.
|
||||
adapter.set_output(MANAGER, "planner", "分工:seat-a 负责需求,seat-b 负责竞品")
|
||||
adapter.set_output(MANAGER, "cross_review", json.dumps({
|
||||
"summary": "审查通过",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [],
|
||||
}))
|
||||
adapter.set_output(MANAGER, "finalizer", "最终结论:市场看好")
|
||||
# Seats emit valid structured output.
|
||||
for s in SEATS:
|
||||
adapter.set_output(s, "expert", json.dumps({"summary": f"{s} 结论", "findings": ["f"]}))
|
||||
|
||||
state = _state()
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(adapter, state).handle("task", ctx)
|
||||
for s in SEATS:
|
||||
await SeatExecutor(adapter, state, s).handle({}, _FakeCtx())
|
||||
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
await ApprovalGateExecutor(adapter, state, svc).handle({}, _FakeCtx())
|
||||
await FinalizerExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
|
||||
# Assert every manager stage produced a distinct seat-run row, all sharing
|
||||
# turn_index (planner=0, cross_review/finalizer advance current_round; the
|
||||
# invariant is that the SAME (agent) has multiple distinct stages).
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
mgr_rows = [r for r in rows if r["agent_id"] == MANAGER]
|
||||
mgr_stages = {r["execution_stage"] for r in mgr_rows}
|
||||
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
|
||||
f"manager stages missing; got {mgr_stages}"
|
||||
# No duplicate four-tuple (the unique constraint would have raised).
|
||||
seen = set()
|
||||
for r in mgr_rows:
|
||||
key = (r["maf_run_id"], r["agent_id"], r["turn_index"], r["execution_stage"])
|
||||
assert key not in seen, f"duplicate four-tuple {key}"
|
||||
seen.add(key)
|
||||
# The final result was produced.
|
||||
assert state.final_result is not None
|
||||
assert state.status == "completed"
|
||||
|
||||
|
||||
# ── 2. expert + format_repair coexist ────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expert_format_repair_coexist(svc):
|
||||
"""A seat that emits invalid JSON triggers exactly one ``format_repair``
|
||||
DeerFlow call; both the expert and the format_repair rows persist for the
|
||||
same (agent, turn) without colliding."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
# First expert call: invalid JSON → triggers repair. The format_repair call
|
||||
# (stage='format_repair') returns valid JSON.
|
||||
adapter.set_output("seat-a", "expert", "not-json-at-all")
|
||||
adapter.set_output("seat-a", "format_repair", json.dumps({"summary": "repaired"}))
|
||||
|
||||
state = _state()
|
||||
state.current_round = 1
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
|
||||
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
|
||||
|
||||
stage_calls = [(a, t, s) for (a, t, s) in adapter.calls if a == "seat-a"]
|
||||
stages = [s for _, _, s in stage_calls]
|
||||
assert "expert" in stages and "format_repair" in stages
|
||||
# Exactly one repair (not a retry loop).
|
||||
assert stages.count("format_repair") == 1
|
||||
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
seat_a_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "seat-a"}
|
||||
assert seat_a_stages == {"expert", "format_repair"}
|
||||
|
||||
|
||||
# ── 3. cross_review + cross_review_repair coexist ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_repair_coexist(svc):
|
||||
"""An invalid cross-review output triggers exactly one ``cross_review_repair``
|
||||
DeerFlow call; cross_review and cross_review_repair rows coexist for the
|
||||
manager."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
adapter.set_output(MANAGER, "cross_review", "totally not json")
|
||||
adapter.set_output(MANAGER, "cross_review_repair", json.dumps({
|
||||
"summary": "repaired review", "confirmed_candidates": [],
|
||||
}))
|
||||
|
||||
state = _state()
|
||||
# Pretend experts already ran so the cross-review barrier sees outcomes.
|
||||
state.seat_outcomes = {s: {"status": "completed"} for s in SEATS}
|
||||
state.current_round = 1
|
||||
|
||||
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
|
||||
mgr_stages = [s for a, _, s in adapter.calls if a == MANAGER]
|
||||
assert "cross_review" in mgr_stages and "cross_review_repair" in mgr_stages
|
||||
assert mgr_stages.count("cross_review_repair") == 1
|
||||
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
mgr_row_stages = {r["execution_stage"] for r in rows if r["agent_id"] == MANAGER}
|
||||
assert {"cross_review", "cross_review_repair"}.issubset(mgr_row_stages)
|
||||
|
||||
# The repaired review was applied (deterministic degradation did NOT happen).
|
||||
assert state.cross_review_result is not None
|
||||
assert state.cross_review_result.get("schema_status") == "repaired"
|
||||
assert state.cross_review_result.get("summary") == "repaired review"
|
||||
|
||||
|
||||
# ── 4. manager present in participants-style config ──────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_role_wired_through_state(svc):
|
||||
"""Sanity: RoundtableState carries manager_agent_id and the planner uses it
|
||||
for the planner seat-turn (not a seat id)."""
|
||||
state = _state()
|
||||
assert state.manager_agent_id == MANAGER
|
||||
assert MANAGER not in state.seat_agent_ids
|
||||
|
||||
adapter = _RecordingAdapter(svc)
|
||||
adapter.set_output(MANAGER, "planner", "plan")
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(adapter, state).handle("task", ctx)
|
||||
|
||||
planner_calls = [(a, s) for a, _, s in adapter.calls if s == "planner"]
|
||||
assert planner_calls == [(MANAGER, "planner")], \
|
||||
"planner stage must run as the MANAGER, not a seat"
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Phase-3 acceptance tests (P0-2): real seat-run identity & idempotency.
|
||||
|
||||
These tests use a REAL SQLite ``MafSeatRunRepository`` (no mocks on the
|
||||
repository) to exercise the Phase-3 four-column unique constraint
|
||||
``(maf_run_id, agent_id, turn_index, execution_stage)`` and the idempotent
|
||||
``get_or_create_seat_run`` acquisition.
|
||||
|
||||
They prove the core Phase-3 principle:
|
||||
|
||||
The business round ``turn_index`` never changes, but every distinct
|
||||
DeerFlow execution gets its own persistent identity via ``execution_stage``.
|
||||
|
||||
Coverage:
|
||||
1. manager planner → cross_review → finalizer within ONE business round
|
||||
coexist (same agent, same turn, different stage).
|
||||
2. expert → format_repair within one business round coexist (same agent,
|
||||
same turn, different stage).
|
||||
3. The same four-tuple second insert is rejected by the DB constraint.
|
||||
4. ``get_or_create_seat_run`` returns the existing row (created=False) instead
|
||||
of launching a duplicate when one already exists.
|
||||
5. Process recovery: an already-completed stage is NOT re-created.
|
||||
6. Idempotent Phase-3 migration + old rows back-filled to ``legacy``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base
|
||||
from deerflow.persistence.maf_roundtable.seat_runs import MafSeatRunRepository
|
||||
|
||||
# ── fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo():
|
||||
"""A real SQLite-backed MafSeatRunRepository over the full schema."""
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
# Seed the parent run row so the FK on seat_runs is satisfiable.
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="s1", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="draft", write_policy="confirm_high_risk",
|
||||
max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="r1", session_id="s1", status="pending",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
r = MafSeatRunRepository(sf)
|
||||
yield r
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── 1. Same business round, different stages coexist (manager path) ─────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_stages_coexist_within_one_business_round(repo):
|
||||
"""planner / cross_review / finalizer all share (run, manager, turn) but
|
||||
differ by execution_stage — all three must insert without conflict."""
|
||||
common = dict(maf_run_id="r1", agent_id="manager-1", turn_index=1)
|
||||
await repo.create_seat_run(execution_stage="planner", **common)
|
||||
await repo.create_seat_run(execution_stage="cross_review", **common)
|
||||
await repo.create_seat_run(execution_stage="finalizer", **common)
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
stages = {r["execution_stage"] for r in rows}
|
||||
assert stages == {"planner", "cross_review", "finalizer"}
|
||||
# All share the SAME business turn_index (never mutated to dodge collision).
|
||||
assert {r["turn_index"] for r in rows} == {1}
|
||||
|
||||
|
||||
# ── 2. expert + format_repair coexist within one business round ─────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expert_and_format_repair_coexist(repo):
|
||||
"""An expert seat turn and its one-shot format-repair retry share the
|
||||
business round but are distinct DeerFlow executions."""
|
||||
common = dict(maf_run_id="r1", agent_id="expert-1", turn_index=1)
|
||||
await repo.create_seat_run(execution_stage="expert", **common)
|
||||
# Same agent, same turn, DIFFERENT stage — must NOT conflict.
|
||||
await repo.create_seat_run(execution_stage="format_repair", **common)
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
assert {r["execution_stage"] for r in rows} == {"expert", "format_repair"}
|
||||
|
||||
|
||||
# ── 3. Same four-tuple second insert is rejected ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_four_tuple_rejected(repo):
|
||||
"""Inserting the exact same (run, agent, turn, stage) twice must raise —
|
||||
the four-column unique constraint enforces at-most-one DeerFlow execution
|
||||
per stage per business round."""
|
||||
common = dict(maf_run_id="r1", agent_id="a1", turn_index=2, execution_stage="expert")
|
||||
await repo.create_seat_run(**common)
|
||||
with pytest.raises(IntegrityError):
|
||||
await repo.create_seat_run(**common)
|
||||
|
||||
|
||||
# ── 4. get_or_create_seat_run is idempotent ─────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_returns_existing_not_duplicate(repo):
|
||||
"""When a seat-run for the four-tuple already exists, get_or_create must
|
||||
return it with created=False rather than launching a duplicate."""
|
||||
key = dict(maf_run_id="r1", agent_id="a2", turn_index=3, execution_stage="cross_review")
|
||||
row1, created1 = await repo.get_or_create_seat_run(**key)
|
||||
assert created1 is True
|
||||
assert row1["execution_stage"] == "cross_review"
|
||||
|
||||
# Second acquisition — existing row returned, NO duplicate created.
|
||||
row2, created2 = await repo.get_or_create_seat_run(**key)
|
||||
assert created2 is False
|
||||
assert row2["id"] == row1["id"]
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
cross_review_rows = [r for r in rows if r["agent_id"] == "a2"]
|
||||
assert len(cross_review_rows) == 1 # exactly one, no duplicate
|
||||
|
||||
|
||||
# ── 5. Process recovery: completed stage is NOT re-created ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_completed_stage_not_recreated(repo):
|
||||
"""After a simulated restart, acquiring the same four-tuple returns the
|
||||
already-persisted row (created=False). The caller must replay the
|
||||
checkpoint's saved stage result instead of starting a new DeerFlow run."""
|
||||
key = dict(maf_run_id="r1", agent_id="a3", turn_index=4, execution_stage="finalizer")
|
||||
row, created = await repo.get_or_create_seat_run(**key)
|
||||
assert created is True
|
||||
await repo.mark_completed(row["id"], checkpoint_id="ckpt-xyz")
|
||||
|
||||
# Simulated restart: fresh acquisition of the same stage.
|
||||
row2, created2 = await repo.get_or_create_seat_run(**key)
|
||||
assert created2 is False
|
||||
assert row2["id"] == row["id"]
|
||||
assert row2["status"] == "completed"
|
||||
assert row2["workflow_checkpoint_id"] == "ckpt-xyz"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_by_four_tuple_lookup(repo):
|
||||
"""Direct four-tuple lookup returns the row or None."""
|
||||
key = dict(maf_run_id="r1", agent_id="a4", turn_index=5, execution_stage="expert")
|
||||
assert await repo.get_by_four_tuple(**key) is None
|
||||
await repo.create_seat_run(**key)
|
||||
found = await repo.get_by_four_tuple(**key)
|
||||
assert found is not None
|
||||
assert found["agent_id"] == "a4"
|
||||
assert found["execution_stage"] == "expert"
|
||||
|
||||
|
||||
# ── 6. Phase-3 migration: idempotent + back-fills legacy ────────────────────
|
||||
|
||||
|
||||
def _load_phase3_migration():
|
||||
here = Path(__file__).resolve().parent.parent
|
||||
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
|
||||
assert spec and spec.loader, "could not load phase3 migration script"
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase3_migration_idempotent_and_backfills_legacy():
|
||||
"""Run the Phase-3 migration against a Phase-2 schema (old 3-tuple
|
||||
constraint), then re-run it (idempotent). Old seat-run rows are
|
||||
back-filled with execution_stage='legacy'."""
|
||||
mod = _load_phase3_migration()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
|
||||
# Build the OLD Phase-2 seat_runs table (3-tuple unique, NO execution_stage).
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_sessions (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
user_id VARCHAR(64) NOT NULL,
|
||||
title VARCHAR(255),
|
||||
task_text TEXT,
|
||||
mode VARCHAR(32),
|
||||
status VARCHAR(32),
|
||||
write_policy VARCHAR(32),
|
||||
max_rounds INTEGER,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text(
|
||||
"CREATE UNIQUE INDEX uq_maf_seat_runs_run_agent_turn "
|
||||
"ON maf_roundtable_seat_runs (maf_run_id, agent_id, turn_index)"
|
||||
))
|
||||
# Seed parent run + a pre-Phase-3 seat-run row.
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_sessions (id, user_id, title, task_text, mode, status, write_policy, max_rounds, created_at, updated_at) "
|
||||
"VALUES ('s1','u1','t','t','structured','draft','confirm_high_risk',8,'2026-01-01','2026-01-01')"
|
||||
))
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_runs (id, session_id, status, created_at, updated_at) "
|
||||
"VALUES ('r1','s1','pending','2026-01-01','2026-01-01')"
|
||||
))
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_seat_runs (id, maf_run_id, agent_id, turn_index, status, created_at, updated_at) "
|
||||
"VALUES ('sr1','r1','a1',1,'completed','2026-01-01','2026-01-01')"
|
||||
))
|
||||
|
||||
# Run the migration.
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
|
||||
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
def _verify(sync_conn):
|
||||
insp = sa_inspect(sync_conn)
|
||||
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
|
||||
assert "execution_stage" in cols
|
||||
# The NEW four-column unique constraint/index is present.
|
||||
new_constraint = mod._find_unique_constraint(
|
||||
insp, "maf_roundtable_seat_runs",
|
||||
expected_cols={"maf_run_id", "agent_id", "turn_index", "execution_stage"},
|
||||
)
|
||||
assert new_constraint is not None, "new four-column unique constraint missing"
|
||||
# The OLD three-column constraint must be gone (rebuilt in SQLite).
|
||||
old_constraint = mod._find_unique_constraint(
|
||||
insp, "maf_roundtable_seat_runs",
|
||||
expected_cols={"maf_run_id", "agent_id", "turn_index"},
|
||||
)
|
||||
assert old_constraint is None, "old three-column constraint still present after migration"
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(_verify)
|
||||
|
||||
# Old row back-filled to 'legacy'.
|
||||
async with sf() as session:
|
||||
row = (await session.execute(
|
||||
text("SELECT execution_stage FROM maf_roundtable_seat_runs WHERE id='sr1'")
|
||||
)).one_or_none()
|
||||
assert row is not None
|
||||
assert row[0] == "legacy"
|
||||
|
||||
# Re-run the migration — must be idempotent (no error).
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=True, logger=mod.logger, changes_ref=[0])
|
||||
|
||||
# After migration, the store can create rows using the new column, and the
|
||||
# constraint allows cross_review + finalizer to coexist for the same
|
||||
# business round (the regression the migration fixes).
|
||||
store = make_maf_roundtable_store(sf)
|
||||
await store.seat_runs.create_seat_run(
|
||||
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="cross_review"
|
||||
)
|
||||
await store.seat_runs.create_seat_run(
|
||||
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="finalizer"
|
||||
)
|
||||
rows = await store.seat_runs.get_by_maf_run("r1")
|
||||
new_mgr_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "mgr"}
|
||||
assert new_mgr_stages == {"cross_review", "finalizer"}
|
||||
|
||||
await engine.dispose()
|
||||
@@ -22,6 +22,7 @@ Additionally covers:
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
@@ -869,8 +870,9 @@ async def test_audit_event_failure_prevents_external_write(store):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deerflow_task_error_marks_run_failed(store):
|
||||
"""P1: when the DeerFlow agent run raises, the MAF run is marked failed
|
||||
(not completed), with a sanitized error message."""
|
||||
"""P3: when a DeerFlow seat turn raises, the seat is soft-failed
|
||||
(``partial_failure``) and the structured run still completes degraded;
|
||||
secrets must not appear in persisted seat / run errors."""
|
||||
svc = _svc(store)
|
||||
s = await svc.create_session(
|
||||
user_id="u1",
|
||||
@@ -890,9 +892,20 @@ async def test_deerflow_task_error_marks_run_failed(store):
|
||||
await runner.execute(s, run)
|
||||
|
||||
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
|
||||
assert final_run["status"] == "failed"
|
||||
assert final_run["status"] == "completed"
|
||||
assert final_run["finished_at"] is not None
|
||||
assert "sk-secret-12345" not in (final_run.get("error_message_safe") or "")
|
||||
ckpt = final_run.get("maf_checkpoint") or {}
|
||||
assert ckpt.get("partial_failure") is True
|
||||
outcome = (ckpt.get("seat_outcomes") or {}).get("boom-seat") or {}
|
||||
assert outcome.get("status") in ("failed", "error")
|
||||
persisted = " ".join(
|
||||
[
|
||||
str(final_run.get("error_message_safe") or ""),
|
||||
str(outcome.get("error") or ""),
|
||||
str(ckpt),
|
||||
]
|
||||
)
|
||||
assert "sk-secret-12345" not in persisted
|
||||
|
||||
|
||||
# ── 13. Secrets never in events / errors / logs ──────────────────────────
|
||||
@@ -1200,3 +1213,107 @@ async def test_capability_snapshot_drives_seat_config(store):
|
||||
|
||||
caps3 = _agent_capabilities_from_snapshot(None, "seat-1")
|
||||
assert caps3 == {}
|
||||
|
||||
|
||||
# ── 22. P1-3: Real WorkflowBuilder manager stages via WorkflowRunner ─────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_workflow_manager_stages_persisted(store):
|
||||
"""P1-3: drive a REAL MAF WorkflowRunner (real WorkflowBuilder, real SQLite
|
||||
repository, real adapter + seat-run store) through a structured run with a
|
||||
manager + 2 seats. Only the runtime client is faked (no real LLM).
|
||||
|
||||
Asserts:
|
||||
- manager has planner, cross_review, finalizer seat-run rows
|
||||
- cross_review.turn_index == finalizer.turn_index
|
||||
- cross_review.execution_stage != finalizer.execution_stage
|
||||
"""
|
||||
svc = _svc(store)
|
||||
s = await svc.create_session(
|
||||
user_id="u1",
|
||||
body={
|
||||
"task_text": "分析市场趋势",
|
||||
"mode": "structured",
|
||||
"manager_agent_id": "manager-1",
|
||||
"participants": [
|
||||
{"agent_id": "manager-1", "role": "manager"},
|
||||
{"agent_id": "seat-a", "role": "seat"},
|
||||
{"agent_id": "seat-b", "role": "seat"},
|
||||
],
|
||||
},
|
||||
)
|
||||
run = await svc.create_run(session_id=s["id"], user_id="u1")
|
||||
|
||||
app = _make_app(store)
|
||||
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
||||
|
||||
manager_responses = iter([
|
||||
"分工:seat-a 负责需求分析,seat-b 负责竞品分析",
|
||||
json.dumps({
|
||||
"summary": "审查通过,证据充分",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [],
|
||||
}),
|
||||
"最终结论:市场看好,建议推进",
|
||||
])
|
||||
seat_responses = {
|
||||
"seat-a": json.dumps({"summary": "需求旺盛", "findings": ["增长10%"]}),
|
||||
"seat-b": json.dumps({"summary": "竞品A领先", "findings": ["份额30%"]}),
|
||||
}
|
||||
|
||||
call_log: list[dict] = []
|
||||
thread_counter = itertools.count()
|
||||
created_threads: list[str] = []
|
||||
latest_agent_by_thread: dict[str, str] = {}
|
||||
|
||||
async def fake_create_thread(principal):
|
||||
thread_id = f"thread-{principal.session_id[:8]}-{next(thread_counter)}"
|
||||
created_threads.append(thread_id)
|
||||
return thread_id
|
||||
|
||||
async def fake_start_agent_run(principal, *, thread_id, agent_id, message, **kwargs):
|
||||
latest_agent_by_thread[thread_id] = agent_id
|
||||
call_log.append({"agent_id": agent_id, "thread_id": thread_id, "message": message})
|
||||
return _FakeRunRecord(run_id=f"df-{agent_id}-{len(call_log)}")
|
||||
|
||||
async def fake_read_messages(principal, thread_id):
|
||||
agent_id = latest_agent_by_thread[thread_id]
|
||||
if agent_id == "manager-1":
|
||||
content = next(manager_responses)
|
||||
else:
|
||||
content = seat_responses[agent_id]
|
||||
return [{"type": "ai", "content": content, "usage_metadata": {"input_tokens": 5, "output_tokens": 5}}]
|
||||
|
||||
runner.adapter._client.create_thread = fake_create_thread
|
||||
runner.adapter._client.start_agent_run = fake_start_agent_run
|
||||
runner.adapter._client.read_thread_messages = fake_read_messages
|
||||
runner.adapter._client.cancel_run = AsyncMock(return_value=True)
|
||||
|
||||
await runner.execute(s, run)
|
||||
|
||||
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
|
||||
assert final_run["status"] == "completed"
|
||||
|
||||
seat_runs = await store.seat_runs.get_by_maf_run(run["id"])
|
||||
mgr_rows = [r for r in seat_runs if r["agent_id"] == "manager-1"]
|
||||
mgr_stages = {r["execution_stage"] for r in mgr_rows}
|
||||
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
|
||||
f"manager stages missing; got {mgr_stages}"
|
||||
|
||||
cr_rows = [r for r in mgr_rows if r["execution_stage"] == "cross_review"]
|
||||
fin_rows = [r for r in mgr_rows if r["execution_stage"] == "finalizer"]
|
||||
assert len(cr_rows) == 1 and len(fin_rows) == 1
|
||||
assert cr_rows[0]["turn_index"] == fin_rows[0]["turn_index"]
|
||||
assert cr_rows[0]["execution_stage"] != fin_rows[0]["execution_stage"]
|
||||
|
||||
assert len(created_threads) == 3
|
||||
assert len(set(created_threads)) == 3
|
||||
|
||||
ckpt = final_run["maf_checkpoint"]
|
||||
assert "最终结论:市场看好" in str(ckpt)
|
||||
assert any(
|
||||
r.get("agent_id") == "seat-a"
|
||||
and (r.get("structured_output") or {}).get("summary") == "需求旺盛"
|
||||
for r in ckpt["turn_results"]
|
||||
)
|
||||
@@ -474,3 +474,43 @@ async def test_identity_from_request_state_not_body(app_and_client):
|
||||
headers={"X-Test-User": "bob"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ── Phase 3 resume / recover ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_and_recover_conflict_when_not_eligible(app_and_client):
|
||||
"""Resume requires waiting_approval; recover requires running|waiting_approval."""
|
||||
app, client, _ = app_and_client
|
||||
app.state.maf_auto_dispatch = False
|
||||
|
||||
resp = await client.post(
|
||||
"/api/maf-roundtable/sessions",
|
||||
json={"task_text": "t"},
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
session_id = resp.json()["id"]
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/sessions/{session_id}/runs",
|
||||
json={},
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
run_id = resp.json()["id"]
|
||||
# Pending runs (auto-dispatch off) are not eligible for resume/recover.
|
||||
assert resp.json()["status"] == "pending"
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/runs/{run_id}/resume",
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/runs/{run_id}/recover",
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Unit tests for seat structured output + one-shot repair."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.seat_output import (
|
||||
build_repair_prompt,
|
||||
parse_seat_output,
|
||||
validate_or_repair_seat_output,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_valid_seat_output():
|
||||
text = '{"summary":"ok","findings":["f1"],"proposed_actions":["do x"]}'
|
||||
model, errors = parse_seat_output(text)
|
||||
assert errors == []
|
||||
assert model is not None
|
||||
assert model.summary == "ok"
|
||||
assert model.findings == ["f1"]
|
||||
|
||||
|
||||
def test_parse_fenced_json():
|
||||
text = '说明\n```json\n{"summary":"s"}\n```\n'
|
||||
model, errors = parse_seat_output(text)
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
|
||||
|
||||
def test_parse_invalid_missing_summary():
|
||||
model, errors = parse_seat_output('{"findings":[]}')
|
||||
assert model is None
|
||||
assert errors
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_ok_no_repair():
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content='{"summary":"fine"}',
|
||||
repair_runner=None,
|
||||
)
|
||||
assert result["schema_status"] == "ok"
|
||||
assert result["repair_attempted"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_shot_repair_success():
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _repair(prompt: str) -> str:
|
||||
calls["n"] += 1
|
||||
assert "SeatTurnOutput" in prompt or "校验" in prompt
|
||||
return '{"summary":"repaired","findings":["x"]}'
|
||||
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content="not json at all",
|
||||
repair_runner=_repair,
|
||||
)
|
||||
assert calls["n"] == 1
|
||||
assert result["schema_status"] == "repaired"
|
||||
assert result["output"]["summary"] == "repaired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repair_still_invalid_stops():
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _repair(prompt: str) -> str:
|
||||
calls["n"] += 1
|
||||
return "still not json"
|
||||
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content="garbage",
|
||||
repair_runner=_repair,
|
||||
)
|
||||
assert calls["n"] == 1
|
||||
assert result["schema_status"] == "schema_invalid"
|
||||
assert result["repair_attempted"] is True
|
||||
|
||||
|
||||
def test_build_repair_prompt_includes_errors():
|
||||
prompt = build_repair_prompt("raw", ["summary: Field required"])
|
||||
assert "summary: Field required" in prompt
|
||||
assert "raw" in prompt
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Phase 3 structured workflow acceptance tests (MAF-03)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.fact_ledger import executed_action_ids, proposed_action_ids
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
CrossReviewExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
)
|
||||
from app.maf_roundtable.workflow_factory import create_workflow
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[Any] = []
|
||||
self.outputs: list[Any] = []
|
||||
|
||||
async def send_message(self, msg: Any) -> None:
|
||||
self.messages.append(msg)
|
||||
|
||||
async def yield_output(self, msg: Any) -> None:
|
||||
self.outputs.append(msg)
|
||||
|
||||
|
||||
class _FakeSeatResponse:
|
||||
def __init__(self, content: str = "", tool_calls: list | None = None, error: str | None = None) -> None:
|
||||
self.content = content
|
||||
self.tool_calls = tool_calls or []
|
||||
self.error = error
|
||||
self.usage = {"input_tokens": 1, "output_tokens": 1}
|
||||
|
||||
|
||||
def _state(seat_ids: list[str] | None = None) -> RoundtableState:
|
||||
seats = seat_ids or ["seat-a", "seat-b", "seat-c"]
|
||||
return RoundtableState(
|
||||
session_id="s1",
|
||||
run_id="r1",
|
||||
task_text="分析市场",
|
||||
mode="structured",
|
||||
model_name=None,
|
||||
reasoning_effort=None,
|
||||
max_rounds=8,
|
||||
write_policy="confirm_high_risk",
|
||||
manager_agent_id=None,
|
||||
seat_agent_ids=seats,
|
||||
thread_map={a: f"t-{a}" for a in seats},
|
||||
capability_snapshot=None,
|
||||
seat_timeout_seconds=5,
|
||||
schema_repair_enabled=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_deterministic_assignments():
|
||||
state = _state()
|
||||
adapter = SimpleNamespace(_emit_critical=AsyncMock())
|
||||
planner = PlannerExecutor(adapter, state)
|
||||
ctx = _FakeCtx()
|
||||
await planner.handle("task", ctx)
|
||||
assert "planner" in state.completed_nodes
|
||||
assert state.plan is not None
|
||||
assert set(state.plan["assignments"]) == {"seat-a", "seat-b", "seat-c"}
|
||||
assert ctx.messages[-1]["decision"] == "parallel_seats"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_skips_when_completed():
|
||||
state = _state()
|
||||
state.mark_completed("planner")
|
||||
state.plan = {"source": "deterministic", "assignments": {}}
|
||||
adapter = SimpleNamespace(run_seat_turn=AsyncMock())
|
||||
planner = PlannerExecutor(adapter, state)
|
||||
ctx = _FakeCtx()
|
||||
await planner.handle("task", ctx)
|
||||
adapter.run_seat_turn.assert_not_called()
|
||||
assert ctx.messages[-1].get("replayed") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seat_partial_failure_does_not_raise():
|
||||
state = _state(["seat-a", "seat-b"])
|
||||
state.plan = {
|
||||
"assignments": {
|
||||
"seat-a": {"instruction": "do a", "role": "A"},
|
||||
"seat-b": {"instruction": "do b", "role": "B"},
|
||||
}
|
||||
}
|
||||
state.current_round = 1
|
||||
|
||||
async def _run(**kwargs):
|
||||
if kwargs["agent_id"] == "seat-a":
|
||||
raise RuntimeError("boom")
|
||||
return _FakeSeatResponse(content='{"summary":"ok-b","findings":["f"]}')
|
||||
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(side_effect=_run),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
|
||||
seat_a = SeatExecutor(adapter, state, "seat-a")
|
||||
seat_b = SeatExecutor(adapter, state, "seat-b")
|
||||
ctx_a, ctx_b = _FakeCtx(), _FakeCtx()
|
||||
await seat_a.handle({}, ctx_a)
|
||||
await seat_b.handle({}, ctx_b)
|
||||
|
||||
assert state.partial_failure is True
|
||||
assert state.seat_outcomes["seat-a"]["status"] in ("failed", "error")
|
||||
assert state.seat_outcomes["seat-b"]["status"] == "completed"
|
||||
assert any(r.get("agent_id") == "seat-b" and r.get("content") for r in state.turn_results)
|
||||
assert "seat_seat-a" in state.completed_nodes
|
||||
assert "seat_seat-b" in state.completed_nodes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seat_schema_one_repair_then_invalid():
|
||||
state = _state(["seat-a"])
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
state.current_round = 1
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _run(**kwargs):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
return _FakeSeatResponse(content="not-json")
|
||||
return _FakeSeatResponse(content="still-bad")
|
||||
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(side_effect=_run),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
seat = SeatExecutor(adapter, state, "seat-a")
|
||||
await seat.handle({}, _FakeCtx())
|
||||
# 1 expert + 1 repair
|
||||
assert calls["n"] == 2
|
||||
row = state.turn_results[-1]
|
||||
assert row["schema_status"] == "schema_invalid"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proposed_not_executed_in_ledger():
|
||||
state = _state(["seat-a"])
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
state.current_round = 1
|
||||
content = (
|
||||
'{"summary":"s","findings":["f"],'
|
||||
'"proposed_actions":[{"action_id":"p1","text":"write"}],'
|
||||
'"executed_actions":[{"action_id":"forge","text":"lie"}]}'
|
||||
)
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=_FakeSeatResponse(content=content)),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
|
||||
assert "p1" in proposed_action_ids(state.fact_ledger)
|
||||
assert "forge" not in executed_action_ids(state.fact_ledger)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_barrier_marks_missing():
|
||||
state = _state(["seat-a", "seat-b"])
|
||||
state.seat_outcomes = {"seat-a": {"status": "completed"}}
|
||||
state.current_round = 1
|
||||
adapter = SimpleNamespace(run_seat_turn=AsyncMock(), _emit_critical=AsyncMock())
|
||||
review = CrossReviewExecutor(adapter, state)
|
||||
await review.handle({}, _FakeCtx())
|
||||
assert state.seat_outcomes["seat-b"]["status"] == "missing"
|
||||
assert state.partial_failure is True
|
||||
assert "cross_review" in state.completed_nodes
|
||||
|
||||
|
||||
def test_structured_factory_requires_planner():
|
||||
with pytest.raises(ValueError):
|
||||
create_workflow(
|
||||
mode="structured",
|
||||
seat_executors=[],
|
||||
approval_gate=object(),
|
||||
finalizer=object(),
|
||||
waiting_terminal=object(),
|
||||
seat_agent_ids=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_skips_completed_seats(tmp_path, monkeypatch):
|
||||
"""completed_nodes must not re-invoke run_seat_turn on recover."""
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'p3r.db'}")
|
||||
async with engine.begin() as conn:
|
||||
import deerflow.persistence.maf_roundtable.model # noqa: F401
|
||||
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
svc = make_maf_service(store)
|
||||
session = await svc.create_session(
|
||||
user_id="u1",
|
||||
body={
|
||||
"task_text": "t",
|
||||
"mode": "structured",
|
||||
"participants": [
|
||||
{"agent_id": "seat-a", "role": "seat"},
|
||||
{"agent_id": "seat-b", "role": "seat"},
|
||||
],
|
||||
},
|
||||
)
|
||||
run = await svc.create_run(session_id=session["id"], user_id="u1")
|
||||
|
||||
# Pretend planner + seat-a already finished; seat-b not yet.
|
||||
state = RoundtableState(
|
||||
session_id=session["id"],
|
||||
run_id=run["id"],
|
||||
task_text="t",
|
||||
mode="structured",
|
||||
model_name=None,
|
||||
reasoning_effort=None,
|
||||
max_rounds=8,
|
||||
write_policy="confirm_high_risk",
|
||||
manager_agent_id=None,
|
||||
seat_agent_ids=["seat-a", "seat-b"],
|
||||
thread_map={"seat-a": "ta", "seat-b": "tb"},
|
||||
capability_snapshot=session.get("capability_snapshot"),
|
||||
seat_timeout_seconds=30,
|
||||
schema_repair_enabled=False,
|
||||
)
|
||||
state.plan = {
|
||||
"source": "deterministic",
|
||||
"assignments": {
|
||||
"seat-a": {"instruction": "a", "role": "A"},
|
||||
"seat-b": {"instruction": "b", "role": "B"},
|
||||
},
|
||||
}
|
||||
state.current_round = 1
|
||||
state.stage = "experts"
|
||||
state.mark_completed("planner")
|
||||
state.mark_completed("seat_seat-a")
|
||||
state.seat_outcomes["seat-a"] = {"status": "completed"}
|
||||
state.turn_results.append({
|
||||
"agent_id": "seat-a",
|
||||
"turn_index": 1,
|
||||
"content": '{"summary":"a"}',
|
||||
"tool_calls": [],
|
||||
"error": None,
|
||||
})
|
||||
await store.runs.update_run(run["id"], {"status": "running", "maf_checkpoint": state.to_dict()})
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
async def _run_seat(**kwargs):
|
||||
aid = kwargs["agent_id"]
|
||||
calls.append(aid)
|
||||
return _FakeSeatResponse(content=f'{{"summary":"{aid}"}}')
|
||||
|
||||
app = SimpleNamespace(state=SimpleNamespace(maf_roundtable_service=svc, checkpointer=None))
|
||||
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
||||
monkeypatch.setattr(runner._adapter, "run_seat_turn", AsyncMock(side_effect=_run_seat))
|
||||
monkeypatch.setattr(runner._adapter, "ensure_threads", AsyncMock(return_value=state.thread_map))
|
||||
monkeypatch.setattr(runner._adapter, "_emit_critical", AsyncMock())
|
||||
|
||||
# Avoid needing a real SDK stream — drive recover path by patching _run_workflow
|
||||
# to invoke remaining executors with skip semantics.
|
||||
async def _fake_run_workflow(workflow, st):
|
||||
# Simulate graph: planner (skip) → seats → cross_review → approval → finalizer
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
SeatExecutor,
|
||||
)
|
||||
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(runner._adapter, st).handle("t", ctx)
|
||||
for aid in st.seat_agent_ids:
|
||||
await SeatExecutor(runner._adapter, st, aid).handle({}, _FakeCtx())
|
||||
await CrossReviewExecutor(runner._adapter, st).handle({}, _FakeCtx())
|
||||
await ApprovalGateExecutor(runner._adapter, st, svc).handle({}, _FakeCtx())
|
||||
await FinalizerExecutor(runner._adapter, st).handle({}, _FakeCtx())
|
||||
st.status = "completed"
|
||||
|
||||
monkeypatch.setattr(runner, "_run_workflow", _fake_run_workflow)
|
||||
try:
|
||||
await runner.recover(run["id"])
|
||||
# seat-a was completed — must not be called; seat-b should run once.
|
||||
assert "seat-a" not in calls
|
||||
assert calls.count("seat-b") == 1
|
||||
finally:
|
||||
await engine.dispose()
|
||||
Reference in new issue
Block a user