feat(maf): complete resilient roundtable workflow
This commit is contained in:
1 parent
bd4131b4e2
commit
21c4c898e7
23 files changed
+4810
-312
No files matched your search
@@ -0,0 +1,108 @@
|
||||
"""Unit tests for the Phase-3 shared fact ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.fact_ledger import (
|
||||
assert_ledger_invariants,
|
||||
empty_ledger,
|
||||
executed_action_ids,
|
||||
merge_seat_output,
|
||||
normalize_ledger,
|
||||
proposed_action_ids,
|
||||
record_action_transition,
|
||||
record_tool_result,
|
||||
)
|
||||
|
||||
|
||||
def test_empty_ledger_has_all_partitions():
|
||||
ledger = empty_ledger()
|
||||
for key in (
|
||||
"confirmed_facts",
|
||||
"evidence",
|
||||
"tool_results",
|
||||
"executed_actions",
|
||||
"pending_actions",
|
||||
"failed_or_unknown_actions",
|
||||
"conflicts",
|
||||
"proposed_actions",
|
||||
):
|
||||
assert key in ledger
|
||||
assert ledger[key] == []
|
||||
|
||||
|
||||
def test_normalize_legacy_checkpoint():
|
||||
legacy = {
|
||||
"confirmed_facts": ["a"],
|
||||
"tool_results": [{"tool_name": "t", "status": "completed"}],
|
||||
"conflicts": [],
|
||||
"pending_actions": [],
|
||||
}
|
||||
n = normalize_ledger(legacy)
|
||||
assert n["confirmed_facts"] == ["a"]
|
||||
assert n["executed_actions"] == []
|
||||
assert n["evidence"] == []
|
||||
|
||||
|
||||
def test_merge_seat_output_keeps_proposed_out_of_executed():
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(
|
||||
ledger,
|
||||
{
|
||||
"summary": "s",
|
||||
"findings": ["fact-1"],
|
||||
"evidence_refs": ["doc:1"],
|
||||
"proposed_actions": [{"text": "create ticket", "action_id": "prop-1"}],
|
||||
"executed_actions": [{"text": "forged", "action_id": "forge-1"}],
|
||||
"conflicts": ["c1"],
|
||||
},
|
||||
agent_id="seat-a",
|
||||
)
|
||||
# Phase-3: seat findings land in candidate_facts (unverified claims), NOT
|
||||
# confirmed_facts. Promotion to confirmed happens only after cross-review.
|
||||
assert ledger["confirmed_facts"] == []
|
||||
assert any(
|
||||
c.get("text") == "fact-1" for c in ledger["candidate_facts"] if isinstance(c, dict)
|
||||
)
|
||||
assert "prop-1" in proposed_action_ids(ledger)
|
||||
assert "forge-1" not in executed_action_ids(ledger)
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_record_action_transition_pending_to_executed():
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(
|
||||
ledger, action_id="a1", tool_name="create_ticket", status="pending", agent_id="s1"
|
||||
)
|
||||
assert len(ledger["pending_actions"]) == 1
|
||||
record_action_transition(
|
||||
ledger, action_id="a1", tool_name="create_ticket", status="executed", agent_id="s1"
|
||||
)
|
||||
assert ledger["pending_actions"] == []
|
||||
assert executed_action_ids(ledger) == {"a1"}
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_record_action_transition_failed_not_executed():
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="pending")
|
||||
record_action_transition(ledger, action_id="a2", tool_name="write_file", status="unknown")
|
||||
assert executed_action_ids(ledger) == set()
|
||||
assert any(e.get("action_id") == "a2" for e in ledger["failed_or_unknown_actions"])
|
||||
|
||||
|
||||
def test_record_tool_result_status_clamp():
|
||||
ledger = empty_ledger()
|
||||
record_tool_result(ledger, tool_name="web_search", status="completed", agent_id="a")
|
||||
record_tool_result(ledger, tool_name="x", status="bogus", agent_id="a")
|
||||
assert ledger["tool_results"][0]["status"] == "completed"
|
||||
assert ledger["tool_results"][1]["status"] == "unknown"
|
||||
assert_ledger_invariants(ledger)
|
||||
|
||||
|
||||
def test_assert_rejects_bad_tool_status():
|
||||
ledger = empty_ledger()
|
||||
ledger["tool_results"].append({"tool_name": "t", "status": "weird"})
|
||||
with pytest.raises(AssertionError):
|
||||
assert_ledger_invariants(ledger)
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Phase-3 acceptance tests for the fact ledger + cross-review wiring.
|
||||
|
||||
Covers P1-2 (candidate_facts separation), P1-3 (approval ledger integrity),
|
||||
and P1-1 (CrossReviewOutput validation + deterministic degradation).
|
||||
|
||||
These are pure-unit tests over the fact_ledger / seat_output / workflow_executors
|
||||
helpers — no DB, no real DeerFlow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.fact_ledger import (
|
||||
candidate_id_map,
|
||||
confirm_candidate_facts,
|
||||
empty_ledger,
|
||||
merge_seat_output,
|
||||
record_action_transition,
|
||||
)
|
||||
from app.maf_roundtable.seat_output import parse_cross_review_output
|
||||
|
||||
# ── P1-2: seat findings → candidate_facts, NOT confirmed_facts ─────────────
|
||||
|
||||
|
||||
def test_seat_findings_land_in_candidate_facts_not_confirmed():
|
||||
"""A seat's `findings` must NEVER be auto-promoted to confirmed_facts —
|
||||
weak model output is a claim, not a fact."""
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["市场增长10%", "竞品A领先"]}, agent_id="seat-1")
|
||||
|
||||
assert len(ledger["candidate_facts"]) == 2
|
||||
assert ledger["confirmed_facts"] == [], "seat findings must not be confirmed facts"
|
||||
|
||||
# Each candidate gets a server-generated candidate_id.
|
||||
for c in ledger["candidate_facts"]:
|
||||
assert "candidate_id" in c and c["candidate_id"]
|
||||
assert c["confirmed"] is False
|
||||
assert c["agent_id"] == "seat-1"
|
||||
assert c["source"] == "seat_finding"
|
||||
|
||||
|
||||
def test_candidate_facts_promoted_only_after_cross_review_confirmation():
|
||||
"""Promotion to confirmed_facts happens only via confirm_candidate_facts,
|
||||
and only for ids that actually exist in candidate_facts AND cite evidence
|
||||
the ledger has backing for."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A", "事实B"]}, agent_id="seat-1")
|
||||
ids = [c["candidate_id"] for c in ledger["candidate_facts"]]
|
||||
assert len(ids) == 2
|
||||
|
||||
# Record real evidence so the confirmation can cite it (Phase-3 P1-1).
|
||||
record_tool_result(
|
||||
ledger, tool_name="web_search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
|
||||
# Model tries to confirm one real id + one HALLUCINATED id, citing the
|
||||
# real evidence for both.
|
||||
tool_ref = "tool:web_search:1"
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[ids[0], "nonexistent-id"],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=[tool_ref],
|
||||
)
|
||||
|
||||
assert len(ledger["confirmed_facts"]) == 1 # only the real one
|
||||
assert ledger["confirmed_facts"][0]["candidate_id"] == ids[0]
|
||||
assert ledger["confirmed_facts"][0]["confirmed_by"] == "manager-1"
|
||||
assert ledger["confirmed_facts"][0]["source"] == "cross_review_confirmed"
|
||||
# The confirmed candidate is marked confirmed=True in place.
|
||||
assert candidate_id_map(ledger)[ids[0]]["confirmed"] is True
|
||||
assert candidate_id_map(ledger)[ids[1]]["confirmed"] is False
|
||||
|
||||
|
||||
def test_candidate_not_promoted_without_ledger_backed_evidence():
|
||||
"""Phase-3 P1-1: confirmation requires evidence that resolves to a ledger
|
||||
entry. A candidate cited with an invented evidence ref (not recorded in the
|
||||
ledger) is REJECTED and stays a candidate."""
|
||||
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
# No evidence recorded in the ledger; the model cites an invented ref.
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[cid],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=["invented-evidence-not-in-ledger"],
|
||||
)
|
||||
|
||||
assert ledger["confirmed_facts"] == [], "candidate promoted with non-ledger evidence"
|
||||
assert candidate_id_map(ledger)[cid]["confirmed"] is False
|
||||
|
||||
|
||||
def test_candidate_not_promoted_with_no_evidence_at_all():
|
||||
"""Phase-3 P1-1: a confirmation that supplies no evidence_refs is rejected."""
|
||||
from app.maf_roundtable.fact_ledger import confirm_candidate_facts
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
confirm_candidate_facts(
|
||||
ledger,
|
||||
confirmed_candidate_ids=[cid],
|
||||
confirmed_by="manager-1",
|
||||
evidence_refs=None,
|
||||
)
|
||||
|
||||
assert ledger["confirmed_facts"] == []
|
||||
assert candidate_id_map(ledger)[cid]["confirmed"] is False
|
||||
|
||||
|
||||
def test_confirm_is_idempotent():
|
||||
"""Confirming the same candidate_id twice does not duplicate it."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
|
||||
ledger = empty_ledger()
|
||||
merge_seat_output(ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
cid = ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
ref = "tool:search:1"
|
||||
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
|
||||
confirm_candidate_facts(ledger, confirmed_candidate_ids=[cid], confirmed_by="m1", evidence_refs=[ref])
|
||||
assert len(ledger["confirmed_facts"]) == 1
|
||||
|
||||
|
||||
def test_empty_ledger_has_candidate_partition():
|
||||
ledger = empty_ledger()
|
||||
assert "candidate_facts" in ledger and ledger["candidate_facts"] == []
|
||||
assert "confirmed_facts" in ledger
|
||||
|
||||
|
||||
# ── P1-3: approval ledger integrity ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pending_action_transitions_on_reject():
|
||||
"""A pending action that is rejected moves to the failed/unknown partition
|
||||
with status 'rejected' — it must not stay in pending and must not appear in
|
||||
executed."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
|
||||
status="pending", agent_id="seat-1")
|
||||
assert len(ledger["pending_actions"]) == 1
|
||||
|
||||
record_action_transition(ledger, action_id="act-1", tool_name="write_file",
|
||||
status="rejected", agent_id="seat-1")
|
||||
|
||||
# Removed from pending.
|
||||
assert not any(p.get("action_id") == "act-1" for p in ledger["pending_actions"])
|
||||
# Present in failed/unknown with status rejected.
|
||||
failed = [f for f in ledger["failed_or_unknown_actions"] if f.get("action_id") == "act-1"]
|
||||
assert len(failed) == 1
|
||||
assert failed[0]["status"] == "rejected"
|
||||
# NOT in executed.
|
||||
assert not any(e.get("action_id") == "act-1" for e in ledger["executed_actions"])
|
||||
|
||||
|
||||
def test_pending_action_transitions_on_execute():
|
||||
"""A pending action that executes successfully moves pending → executed."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
|
||||
status="pending", agent_id="seat-1")
|
||||
record_action_transition(ledger, action_id="act-2", tool_name="create_ticket",
|
||||
status="executed", agent_id="seat-1")
|
||||
assert not any(p.get("action_id") == "act-2" for p in ledger["pending_actions"])
|
||||
assert any(e.get("action_id") == "act-2" and e.get("status") == "executed"
|
||||
for e in ledger["executed_actions"])
|
||||
assert not any(f.get("action_id") == "act-2" for f in ledger["failed_or_unknown_actions"])
|
||||
|
||||
|
||||
def test_failed_action_never_in_executed():
|
||||
"""A failed/unknown action must never leak into executed_actions."""
|
||||
ledger = empty_ledger()
|
||||
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
|
||||
status="pending", agent_id="seat-1")
|
||||
for terminal in ("failed", "unknown"):
|
||||
record_action_transition(ledger, action_id="act-3", tool_name="write_file",
|
||||
status=terminal, agent_id="seat-1")
|
||||
assert not any(e.get("action_id") == "act-3" for e in ledger["executed_actions"]), \
|
||||
f"{terminal} action leaked into executed_actions"
|
||||
|
||||
|
||||
# ── P1-1: CrossReviewOutput parsing ────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_cross_review_output_valid():
|
||||
payload = {
|
||||
"summary": "审查通过",
|
||||
"evidence_ok": True,
|
||||
"gaps": ["缺少 X 证据"],
|
||||
"conflicts": [],
|
||||
"tool_failures": [],
|
||||
"write_status_notes": ["待审批1个"],
|
||||
"recommendations": ["补充X"],
|
||||
"confirmed_candidate_ids": ["c1", "c2"],
|
||||
}
|
||||
model, errors = parse_cross_review_output(json.dumps(payload))
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
assert model.summary == "审查通过"
|
||||
assert model.confirmed_candidate_ids == ["c1", "c2"]
|
||||
|
||||
|
||||
def test_parse_cross_review_confirmed_candidates_requires_evidence():
|
||||
"""Phase-3 P1-1: each ConfirmedCandidate must carry ≥1 evidence ref — an
|
||||
empty list is a schema violation (so the executor repairs instead of
|
||||
silently accepting a bare-id confirmation)."""
|
||||
payload = {
|
||||
"summary": "ok",
|
||||
"confirmed_candidates": [{"candidate_id": "c1", "evidence_refs": []}],
|
||||
}
|
||||
model, errors = parse_cross_review_output(json.dumps(payload))
|
||||
assert model is None
|
||||
assert any("evidence_refs" in e or "confirmed_candidates" in e for e in errors)
|
||||
|
||||
|
||||
def test_parse_cross_review_evidence_ok_defaults_false():
|
||||
"""Phase-3 P1-1: omitting evidence_ok must default to False — a model that
|
||||
forgets the field must never be treated as 'evidence sufficient'."""
|
||||
model, errors = parse_cross_review_output(json.dumps({"summary": "ok"}))
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
assert model.evidence_ok is False
|
||||
|
||||
|
||||
def test_parse_cross_review_output_invalid_json_returns_errors():
|
||||
model, errors = parse_cross_review_output("not json at all")
|
||||
assert model is None
|
||||
assert errors # non-empty readable errors
|
||||
|
||||
|
||||
def test_parse_cross_review_output_missing_summary_invalid():
|
||||
"""summary is required (min_length=1) — a missing summary must fail
|
||||
validation so the CrossReviewExecutor repairs rather than accepting it."""
|
||||
model, errors = parse_cross_review_output(json.dumps({"gaps": ["x"]}))
|
||||
assert model is None
|
||||
assert any("summary" in e for e in errors)
|
||||
|
||||
|
||||
# ── P1-1: CrossReviewExecutor deterministic degradation ────────────────────
|
||||
#
|
||||
# These exercise the executor's _apply_cross_review_output path with a fake
|
||||
# adapter, proving: (a) valid output confirms candidates; (b) invalid output
|
||||
# that the repair also cannot fix degrades deterministically WITHOUT pasting
|
||||
# raw JSON into summary.
|
||||
|
||||
|
||||
def _make_state():
|
||||
"""Minimal RoundtableState for CrossReviewExecutor tests."""
|
||||
from app.maf_roundtable.workflow_executors import RoundtableState
|
||||
|
||||
return RoundtableState(
|
||||
session_id="s1", run_id="r1", task_text="t", mode="structured",
|
||||
model_name=None, reasoning_effort=None, max_rounds=8,
|
||||
write_policy="confirm_high_risk", manager_agent_id="mgr",
|
||||
seat_agent_ids=["seat-1"], thread_map={"mgr": "t-mgr"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_valid_output_confirms_candidates():
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
# Record evidence so the confirmation can bind it (Phase-3 P1-1).
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
# Structured confirmed_candidates path: each candidate cites ledger evidence.
|
||||
raw = json.dumps({
|
||||
"summary": "ok",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert review["source"] == "manager"
|
||||
assert review["schema_status"] == "ok"
|
||||
assert review["summary"] == "ok"
|
||||
# The candidate was promoted to confirmed_facts.
|
||||
assert len(state.fact_ledger["confirmed_facts"]) == 1
|
||||
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
|
||||
# The public confirmation field contains only the server-accepted item;
|
||||
# the model request is retained separately for audit.
|
||||
assert review["confirmed_candidate_ids"] == [candidate_id]
|
||||
assert review["confirmed_candidates"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
|
||||
]
|
||||
assert review["requested_confirmations"] == review["confirmed_candidates"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_rejects_candidate_with_invented_evidence():
|
||||
"""Phase-3 P1-1: a confirmation that cites evidence NOT in the ledger is
|
||||
rejected server-side. The candidate stays unconfirmed, and the rejection is
|
||||
surfaced in the review so the summary can explain it."""
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "ok",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
# Not promoted.
|
||||
assert state.fact_ledger["confirmed_facts"] == []
|
||||
# Rejection recorded.
|
||||
assert review.get("confirmation_rejections"), "rejection not surfaced in review"
|
||||
rej = review["confirmation_rejections"][0]
|
||||
assert rej["candidate_id"] == candidate_id
|
||||
assert rej["reason"] == "evidence_not_in_ledger"
|
||||
assert review["confirmed_candidates"] == []
|
||||
assert review["confirmed_candidate_ids"] == []
|
||||
assert review["requested_confirmations"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": ["invented-not-in-ledger"]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_invalid_then_repair_succeeds():
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
# Fake adapter: repair turn returns valid JSON.
|
||||
good_repair = json.dumps({"summary": "repaired ok", "confirmed_candidate_ids": []})
|
||||
fake_adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content=good_repair)),
|
||||
)
|
||||
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
|
||||
review: dict = {}
|
||||
# First output is garbage; repair returns valid.
|
||||
await executor._apply_cross_review_output(review, "totally not json", "mgr")
|
||||
|
||||
assert review["source"] == "manager"
|
||||
assert review["schema_status"] == "repaired"
|
||||
assert review["summary"] == "repaired ok"
|
||||
fake_adapter.run_seat_turn.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_invalid_and_repair_fails_degrades_deterministically():
|
||||
"""When both the original and the repair output are unparseable, the
|
||||
review MUST degrade deterministically: raw JSON is never pasted into
|
||||
summary, and the review is marked schema_invalid."""
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
# Repair also returns garbage.
|
||||
fake_adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=SimpleNamespace(content="still garbage")),
|
||||
)
|
||||
executor = CrossReviewExecutor(adapter=fake_adapter, state=state)
|
||||
review: dict = {}
|
||||
await executor._apply_cross_review_output(review, "garbage first output", "mgr")
|
||||
|
||||
assert review["source"] == "deterministic"
|
||||
assert review["schema_status"] == "schema_invalid"
|
||||
assert review.get("raw_response_ignored") is True
|
||||
# summary must NOT be the raw model JSON.
|
||||
assert review.get("summary", "") != "garbage first output"
|
||||
|
||||
|
||||
# ── P1-2: evidence_ok=False blocks all promotion ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_ok_false_blocks_promotion_even_with_valid_refs():
|
||||
"""Phase-3 P1-2: when the model says evidence_ok=False, no candidate may
|
||||
be promoted even if it cites valid ledger-backed evidence refs. The
|
||||
attempted confirmations are recorded as rejections."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实A"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "证据不充分",
|
||||
"evidence_ok": False,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert state.fact_ledger["confirmed_facts"] == []
|
||||
assert candidate_id_map(state.fact_ledger)[candidate_id]["confirmed"] is False
|
||||
assert review.get("confirmation_rejections")
|
||||
rej = review["confirmation_rejections"][0]
|
||||
assert rej["candidate_id"] == candidate_id
|
||||
assert rej["reason"] == "review_did_not_confirm_evidence"
|
||||
assert review["confirmed_candidates"] == []
|
||||
assert review["confirmed_candidate_ids"] == []
|
||||
assert review["requested_confirmations"] == [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evidence_ok_true_allows_promotion_with_valid_refs():
|
||||
"""Phase-3 P1-2: when evidence_ok=True AND evidence refs are valid, the
|
||||
candidate IS promoted."""
|
||||
from app.maf_roundtable.fact_ledger import record_tool_result
|
||||
from app.maf_roundtable.workflow_executors import CrossReviewExecutor
|
||||
|
||||
state = _make_state()
|
||||
merge_seat_output(state.fact_ledger, {"findings": ["事实B"]}, agent_id="seat-1")
|
||||
candidate_id = state.fact_ledger["candidate_facts"][0]["candidate_id"]
|
||||
record_tool_result(
|
||||
state.fact_ledger, tool_name="search", status="completed",
|
||||
agent_id="seat-1", turn_index=1,
|
||||
)
|
||||
tool_ref = "tool:search:1"
|
||||
|
||||
executor = CrossReviewExecutor(adapter=SimpleNamespace(), state=state)
|
||||
review: dict = {}
|
||||
raw = json.dumps({
|
||||
"summary": "证据充分",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [
|
||||
{"candidate_id": candidate_id, "evidence_refs": [tool_ref]},
|
||||
],
|
||||
})
|
||||
await executor._apply_cross_review_output(review, raw, "mgr")
|
||||
|
||||
assert len(state.fact_ledger["confirmed_facts"]) == 1
|
||||
assert state.fact_ledger["confirmed_facts"][0]["candidate_id"] == candidate_id
|
||||
@@ -0,0 +1,619 @@
|
||||
"""Phase-3 Round-2 acceptance tests (P0-1): race-free idempotency & replay.
|
||||
|
||||
These prove the Phase-3 promise: a distinct DeerFlow execution (run, agent,
|
||||
turn, execution_stage) is started at most once, even under concurrency,
|
||||
retries, recovery, or a partial crash. They use a REAL SQLite-backed store +
|
||||
the REAL ``DeerFlowAgentAdapter`` (only ``_execute_run`` is faked, so the
|
||||
adapter's get_or_create / status-branching / snapshot logic is exercised
|
||||
end-to-end against a real DB).
|
||||
|
||||
Coverage:
|
||||
- Two coroutines acquiring the same four-tuple concurrently → exactly one wins
|
||||
(created=True), the other reads the existing row (created=False). This is the
|
||||
race that "query-then-insert" lost; the atomic insert-then-read wins it.
|
||||
- A double-call of ``run_seat_turn`` after completion does NOT start a second
|
||||
DeerFlow run — it replays the saved ``result_snapshot``.
|
||||
- A call while the stage is pending/running returns an in-flight marker and
|
||||
does NOT start a second DeerFlow run.
|
||||
- A failed/cancelled stage is terminal and never auto-retried.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.deerflow_agent_adapter import DeerFlowAgentAdapter, SeatResponse
|
||||
from app.maf_roundtable.runtime_client import DeerFlowRuntimeClient
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def svc(tmp_path):
|
||||
# IMPORTANT: a file-based SQLite is required for the concurrency test. The
|
||||
# in-memory engine (``sqlite+aiosqlite://``) multiplexes all sessions over a
|
||||
# SINGLE connection, so two coroutines in ``asyncio.gather`` share one
|
||||
# transaction — the loser's rollback would void the winner's uncommitted
|
||||
# insert, making a true race impossible to simulate. A file DB gives each
|
||||
# ``async with self._sf()`` its own connection (production behavior), so the
|
||||
# four-column unique constraint is the real arbiter.
|
||||
db_path = tmp_path / "idempotency.db"
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
service = make_maf_service(store)
|
||||
|
||||
# Seed a session + run so the seat_runs FK is satisfiable.
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="sess-1", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="running", write_policy="auto",
|
||||
max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="run-1", session_id="sess-1", status="running",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
yield service
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _adapter(svc) -> DeerFlowAgentAdapter:
|
||||
app = MagicMock()
|
||||
app.state.checkpointer = AsyncMock()
|
||||
app.state.thread_store = AsyncMock()
|
||||
app.state.stream_bridge = None
|
||||
app.state.run_manager = AsyncMock()
|
||||
client = DeerFlowRuntimeClient(app)
|
||||
return DeerFlowAgentAdapter(client, service=svc, user_id="u1")
|
||||
|
||||
|
||||
# ── 1. Concurrent acquisition: exactly one wins ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_get_or_create_one_winner(svc):
|
||||
"""Two coroutines acquire the same four-tuple at (near-)the same time. The
|
||||
atomic insert-then-read means the DB constraint picks a single winner; the
|
||||
loser reads the winner's row. Both calls return, neither raises, and there
|
||||
is exactly ONE row for the four-tuple."""
|
||||
repo = svc.store.seat_runs
|
||||
key = dict(maf_run_id="run-1", agent_id="a1", turn_index=1, execution_stage="expert")
|
||||
|
||||
(ar, ac), (br, bc) = await asyncio.gather(
|
||||
repo.get_or_create_seat_run(**key),
|
||||
repo.get_or_create_seat_run(**key),
|
||||
)
|
||||
assert (ac, bc) in {(True, False), (False, True)}, "exactly one winner; the other reads existing"
|
||||
assert ar["id"] == br["id"]
|
||||
rows = await repo.get_by_maf_run("run-1")
|
||||
assert len([r for r in rows if r["agent_id"] == "a1" and r["execution_stage"] == "expert"]) == 1
|
||||
|
||||
|
||||
# ── 2. Double-call after completion replays the snapshot ────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_double_call_after_completion_does_not_re_run(svc):
|
||||
"""A second ``run_seat_turn`` for an already-completed stage must replay the
|
||||
saved ``result_snapshot`` and NOT call ``_execute_run`` again. This is the
|
||||
core recovery guarantee: a restart or retry never re-invokes the model."""
|
||||
adapter = _adapter(svc)
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"],
|
||||
turn_index=kwargs["turn_index"],
|
||||
content="first-run-output",
|
||||
tool_calls=[{"name": "search", "args": {}, "id": "tc1"}],
|
||||
usage={"input_tokens": 5, "output_tokens": 5},
|
||||
metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a2", turn_index=2,
|
||||
execution_stage="expert", message="q", thread_id="t-a2",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1
|
||||
assert first.content == "first-run-output"
|
||||
|
||||
# Second call — completed stage. Must replay, NOT re-run.
|
||||
second = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1, "completed stage was re-run instead of replayed"
|
||||
assert second.content == "first-run-output"
|
||||
assert second.error is None
|
||||
|
||||
|
||||
# ── 3. In-flight stage: second caller waits and replays ─────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_running_stage_second_caller_waits_and_replays(svc):
|
||||
"""When a stage is pending/running (another caller owns it), a second
|
||||
``run_seat_turn`` must WAIT for the first to complete, then replay the
|
||||
same snapshot. ``_execute_run`` is called exactly once."""
|
||||
adapter = _adapter(svc)
|
||||
execute_started = asyncio.Event()
|
||||
release_execute = asyncio.Event()
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _blocking_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
execute_started.set()
|
||||
await release_execute.wait()
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="done-content", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
|
||||
usage={"input_tokens": 1}, metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a3", turn_index=3,
|
||||
execution_stage="cross_review", message="q", thread_id="t-a3",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first_task = asyncio.create_task(adapter.run_seat_turn(**common))
|
||||
await execute_started.wait()
|
||||
|
||||
second_task = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
release_execute.set()
|
||||
first = await first_task
|
||||
second = await second_task
|
||||
|
||||
assert calls["execute"] == 1, "in-flight stage started a second DeerFlow run"
|
||||
assert first.content == "done-content"
|
||||
assert second.content == "done-content"
|
||||
assert second.error is None
|
||||
|
||||
|
||||
# ── 4. Failed/cancelled stage is terminal ────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_stage_not_auto_retried(svc):
|
||||
"""A previously-failed stage is terminal; a new ``run_seat_turn`` surfaces
|
||||
a terminal marker and never auto-retries (no silent re-run)."""
|
||||
adapter = _adapter(svc)
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _failing_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
raise RuntimeError("model blew up")
|
||||
|
||||
adapter._execute_run = _failing_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a4", turn_index=4,
|
||||
execution_stage="finalizer", message="q", thread_id="t-a4",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
first = await adapter.run_seat_turn(**common)
|
||||
assert first.error # failed
|
||||
assert calls["execute"] == 1
|
||||
|
||||
second = await adapter.run_seat_turn(**common)
|
||||
assert calls["execute"] == 1, "failed stage was auto-retried"
|
||||
assert second.error is not None
|
||||
assert second.metadata is not None and second.metadata.get("terminal") == "failed"
|
||||
|
||||
|
||||
# ── 5. Four-tuple replay snapshot persists sanitized content ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_result_snapshot_persisted_on_completion(svc):
|
||||
"""The sanitized result snapshot is written atomically with the completed
|
||||
transition so recovery can replay it even if the workflow checkpoint was
|
||||
not yet written."""
|
||||
adapter = _adapter(svc)
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="snapshot-me", tool_calls=[{"name": "t", "args": {}, "id": "x"}],
|
||||
usage={"input_tokens": 1}, metadata={"thread_id": "t"},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a5", turn_index=5,
|
||||
execution_stage="expert", message="q", thread_id="t-a5",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
|
||||
row = await svc.store.seat_runs.get_by_four_tuple(
|
||||
maf_run_id="run-1", agent_id="a5", turn_index=5, execution_stage="expert",
|
||||
)
|
||||
assert row is not None
|
||||
assert row["status"] == "completed"
|
||||
snap = row["result_snapshot"]
|
||||
assert snap is not None
|
||||
assert snap["content"] == "snapshot-me"
|
||||
assert snap["tool_calls"][0]["name"] == "t"
|
||||
|
||||
|
||||
# ── 6. Invalid execution_stage rejected ──────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_execution_stage_rejected(svc):
|
||||
"""A typo'd execution_stage (e.g. 'exprt') is rejected at the repository —
|
||||
it would otherwise bypass the four-column idempotency key by creating a
|
||||
brand-new stage."""
|
||||
from deerflow.persistence.maf_roundtable.seat_runs import InvalidExecutionStageError
|
||||
|
||||
repo = svc.store.seat_runs
|
||||
with pytest.raises(InvalidExecutionStageError):
|
||||
await repo.get_or_create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a6", turn_index=6,
|
||||
execution_stage="exprt", # typo
|
||||
)
|
||||
with pytest.raises(InvalidExecutionStageError):
|
||||
await repo.create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a6", turn_index=6,
|
||||
execution_stage="expert2", # not in the closed enum
|
||||
)
|
||||
|
||||
|
||||
# ── 7. Migration branching: constraint vs index drop ─────────────────────────
|
||||
|
||||
|
||||
def _load_phase3_migration():
|
||||
here = Path(__file__).resolve().parent.parent
|
||||
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
|
||||
assert spec and spec.loader, "could not load phase3 migration script"
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def test_migration_ddl_drop_branches_on_kind():
|
||||
"""The migration must emit DROP CONSTRAINT for a table constraint and DROP
|
||||
INDEX for a unique index — the Phase-2 deployment created the constraint as
|
||||
a CREATE UNIQUE INDEX, so calling DROP CONSTRAINT on it failed."""
|
||||
mod = _load_phase3_migration()
|
||||
# A constraint-kind object → DROP CONSTRAINT.
|
||||
drop_constraint_sql = mod._ddl_drop_constraint("postgresql", "uq_maf_seat_runs_run_agent_turn")
|
||||
assert "DROP CONSTRAINT" in drop_constraint_sql
|
||||
# An index-kind object → DROP INDEX.
|
||||
drop_index_sql = mod._ddl_drop_index("uq_maf_seat_runs_run_agent_turn")
|
||||
assert "DROP INDEX" in drop_index_sql
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_sqlite_rebuild_restores_deerflow_run_id_index():
|
||||
"""The SQLite rebuild must recreate the ``deerflow_run_id`` index — without
|
||||
it, cancel-by-deerflow-run-id lookups silently degrade after a restart."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
|
||||
joined = "\n".join(rebuild)
|
||||
assert "deerflow_run_id" in joined, "rebuild dropped the deerflow_run_id index"
|
||||
assert "ck_maf_seat_runs_execution_stage" in joined
|
||||
assert "execution_stage IN" in joined
|
||||
|
||||
|
||||
# ── 8. P0-1: Two adapters same stage — wait, not partial_failure ─────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_adapters_same_stage_wait_not_partial_failure(svc):
|
||||
"""Two adapter calls for the same four-tuple: the first blocks executing,
|
||||
the second waits. After release, both get the same snapshot. _execute_run
|
||||
is called exactly once. Neither sets partial_failure."""
|
||||
adapter = _adapter(svc)
|
||||
execute_started = asyncio.Event()
|
||||
release_execute = asyncio.Event()
|
||||
calls = {"execute": 0}
|
||||
|
||||
async def _blocking_execute(**kwargs):
|
||||
calls["execute"] += 1
|
||||
execute_started.set()
|
||||
await release_execute.wait()
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="shared-result", tool_calls=[], usage={"input_tokens": 2},
|
||||
metadata={"thread_id": kwargs["thread_id"]},
|
||||
)
|
||||
|
||||
adapter._execute_run = _blocking_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
common = dict(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-wait", turn_index=7,
|
||||
execution_stage="expert", message="q", thread_id="t-wait",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
task1 = asyncio.create_task(adapter.run_seat_turn(**common))
|
||||
await execute_started.wait()
|
||||
|
||||
task2 = asyncio.create_task(adapter.run_seat_turn(**common, wait_timeout_seconds=10))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
release_execute.set()
|
||||
r1, r2 = await asyncio.gather(task1, task2)
|
||||
|
||||
assert calls["execute"] == 1
|
||||
assert r1.content == "shared-result"
|
||||
assert r2.content == "shared-result"
|
||||
assert r1.error is None
|
||||
assert r2.error is None
|
||||
|
||||
|
||||
# ── 9. P0-1: Empty snapshot defense ──────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_with_empty_snapshot_returns_error(svc):
|
||||
"""A completed seat run with no result_snapshot must NOT produce a fake
|
||||
success — it returns an explicit error requiring manual recovery."""
|
||||
adapter = _adapter(svc)
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
seat_run = await svc.store.seat_runs.create_seat_run(
|
||||
maf_run_id="run-1", agent_id="a-empty", turn_index=8, execution_stage="expert",
|
||||
)
|
||||
await svc.store.seat_runs.mark_completed(seat_run["id"], result_snapshot=None)
|
||||
|
||||
result = await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-empty", turn_index=8,
|
||||
execution_stage="expert", message="q", thread_id="t-empty",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
assert result.error is not None
|
||||
assert "no replayable snapshot" in result.error
|
||||
assert result.content == ""
|
||||
|
||||
|
||||
# ── 10. P0-2: Snapshot sanitization before persist ───────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_result_snapshot_sanitized_before_persist(svc):
|
||||
"""Secrets in tool_calls args and metadata must be redacted in the
|
||||
persisted snapshot. Safe fields are preserved."""
|
||||
adapter = _adapter(svc)
|
||||
|
||||
async def _fake_execute(**kwargs):
|
||||
return SeatResponse(
|
||||
agent_id=kwargs["agent_id"], turn_index=kwargs["turn_index"],
|
||||
content="analysis complete",
|
||||
tool_calls=[{
|
||||
"name": "request_api",
|
||||
"args": {
|
||||
"api_key": "secret-api-key",
|
||||
"password": "secret-password",
|
||||
"normal": "safe",
|
||||
},
|
||||
"id": "tc1",
|
||||
}],
|
||||
usage={"input_tokens": 1},
|
||||
metadata={"authorization": "Bearer secret-token", "thread_id": "t"},
|
||||
)
|
||||
|
||||
adapter._execute_run = _fake_execute # type: ignore[method-assign]
|
||||
adapter._emit_critical = AsyncMock() # type: ignore[method-assign]
|
||||
adapter._emit = AsyncMock() # type: ignore[method-assign]
|
||||
|
||||
await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
|
||||
execution_stage="expert", message="q", thread_id="t-san",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
|
||||
row = await svc.store.seat_runs.get_by_four_tuple(
|
||||
maf_run_id="run-1", agent_id="a-san", turn_index=9, execution_stage="expert",
|
||||
)
|
||||
assert row is not None
|
||||
snap = row["result_snapshot"]
|
||||
assert snap is not None
|
||||
snap_str = str(snap)
|
||||
assert "secret-api-key" not in snap_str
|
||||
assert "secret-password" not in snap_str
|
||||
assert "secret-token" not in snap_str
|
||||
assert snap["tool_calls"][0]["args"]["normal"] == "safe"
|
||||
|
||||
replayed = await adapter.run_seat_turn(
|
||||
session_id="sess-1", run_id="run-1", agent_id="a-san", turn_index=9,
|
||||
execution_stage="expert", message="q", thread_id="t-san",
|
||||
task_text="q", stage_instruction="",
|
||||
)
|
||||
assert replayed.content == "analysis complete"
|
||||
assert "secret-api-key" not in str(replayed.tool_calls)
|
||||
|
||||
|
||||
# ── 11. P1-1: Migration intermediate state tests ─────────────────────────────
|
||||
|
||||
|
||||
def test_migration_rebuild_preserves_existing_stage_and_snapshot():
|
||||
"""When the source table already has execution_stage and result_snapshot,
|
||||
the rebuild must copy them verbatim — not overwrite with 'legacy'/NULL."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=True, source_has_snapshot=True)
|
||||
insert_sql = rebuild[1]
|
||||
assert "execution_stage" in insert_sql
|
||||
assert "'legacy'" not in insert_sql.split("SELECT")[1]
|
||||
assert "result_snapshot" in insert_sql
|
||||
assert "NULL" not in insert_sql.split("SELECT")[1].split("FROM")[0].replace("result_snapshot", "")
|
||||
|
||||
|
||||
def test_migration_rebuild_from_phase2_uses_defaults():
|
||||
"""When the source table has neither stage nor snapshot (Phase-2), the
|
||||
rebuild uses 'legacy' and NULL."""
|
||||
mod = _load_phase3_migration()
|
||||
rebuild = mod._sqlite_rebuild_seat_runs(source_has_stage=False, source_has_snapshot=False)
|
||||
insert_sql = rebuild[1]
|
||||
select_part = insert_sql.split("SELECT")[1].split("FROM")[0]
|
||||
assert "'legacy'" in select_part
|
||||
assert "NULL" in select_part
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_idempotent_double_run(tmp_path):
|
||||
"""Running the migration twice on the same SQLite DB succeeds both times."""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "double_migrate.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_maf_seat_runs_run_agent_turn UNIQUE (maf_run_id, agent_id, turn_index)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
INSERT INTO maf_roundtable_seat_runs
|
||||
(id, maf_run_id, agent_id, turn_index, status, created_at, updated_at)
|
||||
VALUES ('sr1', 'run-1', 'a1', 1, 'completed', '2026-01-01', '2026-01-01')
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
mod = _load_phase3_migration()
|
||||
|
||||
for _ in range(2):
|
||||
with engine.connect() as sync_conn:
|
||||
mod._do_migrate_sync(
|
||||
sync_conn, dry_run=False, logger=logger, changes_ref=[0]
|
||||
)
|
||||
sync_conn.commit()
|
||||
|
||||
insp = inspect(engine)
|
||||
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
|
||||
assert "execution_stage" in cols
|
||||
assert "result_snapshot" in cols
|
||||
|
||||
with engine.connect() as conn:
|
||||
row = conn.execute(text("SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr1'")).fetchone()
|
||||
assert row[0] == "legacy"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_intermediate_preserves_expert_stage(tmp_path):
|
||||
"""An intermediate table (has stage + 4-col unique, but no snapshot/check)
|
||||
must preserve existing execution_stage='expert' and result_snapshot data
|
||||
after rebuild."""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "intermediate.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
result_snapshot TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_maf_seat_runs_run_agent_turn_stage UNIQUE (maf_run_id, agent_id, turn_index, execution_stage)
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
INSERT INTO maf_roundtable_seat_runs
|
||||
(id, maf_run_id, agent_id, turn_index, execution_stage, status, result_snapshot, created_at, updated_at)
|
||||
VALUES ('sr2', 'run-1', 'a1', 1, 'expert', 'completed', '{"content": "keep-me"}', '2026-01-01', '2026-01-01')
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
mod = _load_phase3_migration()
|
||||
|
||||
with engine.connect() as sync_conn:
|
||||
mod._do_migrate_sync(sync_conn, dry_run=False, logger=logger, changes_ref=[0])
|
||||
sync_conn.commit()
|
||||
|
||||
with engine.connect() as conn:
|
||||
row = conn.execute(text(
|
||||
"SELECT execution_stage, result_snapshot FROM maf_roundtable_seat_runs WHERE id='sr2'"
|
||||
)).fetchone()
|
||||
assert row[0] == "expert", f"stage corrupted to {row[0]!r}"
|
||||
assert "keep-me" in (row[1] or ""), f"snapshot corrupted to {row[1]!r}"
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Phase-3 Round-2 acceptance tests (P0-3): real manager workflow coexistence.
|
||||
|
||||
Drives the REAL workflow executors (PlannerExecutor → SeatExecutor ×N →
|
||||
CrossReviewExecutor → ApprovalGateExecutor → FinalizerExecutor) through a
|
||||
manager + 2-seat structured run, with a fake DeerFlow adapter whose
|
||||
``run_seat_turn`` records each call's ``execution_stage`` into a REAL seat-run
|
||||
store. This proves the Phase-3 disambiguation invariants end-to-end:
|
||||
|
||||
- manager planner / cross_review / finalizer all share the SAME business round
|
||||
(turn_index) but occupy distinct seat-run rows via ``execution_stage`` — they
|
||||
must coexist without a four-tuple collision.
|
||||
- an expert seat turn and its one-shot ``format_repair`` retry coexist for the
|
||||
same (agent, turn).
|
||||
- invalid expert output triggers exactly one ``format_repair`` DeerFlow call,
|
||||
and both rows persist.
|
||||
- invalid cross-review output triggers exactly one ``cross_review_repair``
|
||||
DeerFlow call, and ``cross_review`` + ``cross_review_repair`` rows coexist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.deerflow_agent_adapter import SeatResponse
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
)
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base, MafRoundtableRunRow, MafRoundtableSessionRow
|
||||
|
||||
MANAGER = "manager-1"
|
||||
SEATS = ["seat-a", "seat-b"]
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[object] = []
|
||||
self.outputs: list[object] = []
|
||||
|
||||
async def send_message(self, msg: object) -> None:
|
||||
self.messages.append(msg)
|
||||
|
||||
async def yield_output(self, msg: object) -> None:
|
||||
self.outputs.append(msg)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def svc():
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
service = make_maf_service(store)
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="sess-1", user_id="u1", title="t", task_text="分析市场",
|
||||
mode="structured", status="running", write_policy="auto",
|
||||
manager_agent_id=MANAGER, max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="run-1", session_id="sess-1", status="running",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
yield service
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _state() -> RoundtableState:
|
||||
return RoundtableState(
|
||||
session_id="sess-1", run_id="run-1", task_text="分析市场",
|
||||
mode="structured", model_name=None, reasoning_effort=None,
|
||||
max_rounds=8, write_policy="auto", manager_agent_id=MANAGER,
|
||||
seat_agent_ids=SEATS,
|
||||
thread_map={MANAGER: "t-mgr", "seat-a": "t-a", "seat-b": "t-b"},
|
||||
capability_snapshot=None, seat_timeout_seconds=5, schema_repair_enabled=True,
|
||||
)
|
||||
|
||||
|
||||
def _resp(content: str) -> SeatResponse:
|
||||
return SeatResponse(
|
||||
agent_id="x", turn_index=0, content=content,
|
||||
tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1},
|
||||
)
|
||||
|
||||
|
||||
class _RecordingAdapter:
|
||||
"""Fake adapter that (a) records every run_seat_turn's execution_stage and
|
||||
(b) writes a REAL seat-run row for each call so the four-tuple coexistence
|
||||
invariant is checked against the actual DB."""
|
||||
|
||||
def __init__(self, service) -> None:
|
||||
self._service = service
|
||||
self.calls: list[tuple[str, str, int, str]] = [] # (agent, turn, stage)
|
||||
self._seat_outputs: dict[tuple[str, str], str] = {} # (agent, stage) -> content
|
||||
self._user_id = "u1"
|
||||
self._emit_critical = AsyncMock()
|
||||
self._emit = AsyncMock()
|
||||
|
||||
def set_output(self, agent_id: str, execution_stage: str, content: str) -> None:
|
||||
self._seat_outputs[(agent_id, execution_stage)] = content
|
||||
|
||||
async def run_seat_turn(self, **kwargs):
|
||||
agent_id = kwargs["agent_id"]
|
||||
turn_index = kwargs["turn_index"]
|
||||
stage = kwargs.get("execution_stage", "expert")
|
||||
self.calls.append((agent_id, turn_index, stage))
|
||||
# Persist a seat-run row so coexistence is verified against the real DB.
|
||||
await self._service.store.seat_runs.create_seat_run(
|
||||
maf_run_id=kwargs["run_id"], agent_id=agent_id,
|
||||
turn_index=turn_index, execution_stage=stage,
|
||||
)
|
||||
content = self._seat_outputs.get((agent_id, stage), '{"summary":"ok"}')
|
||||
return _resp(content)
|
||||
|
||||
|
||||
# ── 1. Full structured run: manager stages coexist ───────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_structured_run_manager_stages_coexist(svc):
|
||||
"""A complete planner→experts→cross_review→approval→finalizer run must
|
||||
persist manager planner/cross_review/finalizer rows that ALL share the same
|
||||
business round (turn_index) but differ by execution_stage — no collision."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
# Manager emits valid JSON at each stage.
|
||||
adapter.set_output(MANAGER, "planner", "分工:seat-a 负责需求,seat-b 负责竞品")
|
||||
adapter.set_output(MANAGER, "cross_review", json.dumps({
|
||||
"summary": "审查通过",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [],
|
||||
}))
|
||||
adapter.set_output(MANAGER, "finalizer", "最终结论:市场看好")
|
||||
# Seats emit valid structured output.
|
||||
for s in SEATS:
|
||||
adapter.set_output(s, "expert", json.dumps({"summary": f"{s} 结论", "findings": ["f"]}))
|
||||
|
||||
state = _state()
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(adapter, state).handle("task", ctx)
|
||||
for s in SEATS:
|
||||
await SeatExecutor(adapter, state, s).handle({}, _FakeCtx())
|
||||
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
await ApprovalGateExecutor(adapter, state, svc).handle({}, _FakeCtx())
|
||||
await FinalizerExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
|
||||
# Assert every manager stage produced a distinct seat-run row, all sharing
|
||||
# turn_index (planner=0, cross_review/finalizer advance current_round; the
|
||||
# invariant is that the SAME (agent) has multiple distinct stages).
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
mgr_rows = [r for r in rows if r["agent_id"] == MANAGER]
|
||||
mgr_stages = {r["execution_stage"] for r in mgr_rows}
|
||||
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
|
||||
f"manager stages missing; got {mgr_stages}"
|
||||
# No duplicate four-tuple (the unique constraint would have raised).
|
||||
seen = set()
|
||||
for r in mgr_rows:
|
||||
key = (r["maf_run_id"], r["agent_id"], r["turn_index"], r["execution_stage"])
|
||||
assert key not in seen, f"duplicate four-tuple {key}"
|
||||
seen.add(key)
|
||||
# The final result was produced.
|
||||
assert state.final_result is not None
|
||||
assert state.status == "completed"
|
||||
|
||||
|
||||
# ── 2. expert + format_repair coexist ────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expert_format_repair_coexist(svc):
|
||||
"""A seat that emits invalid JSON triggers exactly one ``format_repair``
|
||||
DeerFlow call; both the expert and the format_repair rows persist for the
|
||||
same (agent, turn) without colliding."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
# First expert call: invalid JSON → triggers repair. The format_repair call
|
||||
# (stage='format_repair') returns valid JSON.
|
||||
adapter.set_output("seat-a", "expert", "not-json-at-all")
|
||||
adapter.set_output("seat-a", "format_repair", json.dumps({"summary": "repaired"}))
|
||||
|
||||
state = _state()
|
||||
state.current_round = 1
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
|
||||
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
|
||||
|
||||
stage_calls = [(a, t, s) for (a, t, s) in adapter.calls if a == "seat-a"]
|
||||
stages = [s for _, _, s in stage_calls]
|
||||
assert "expert" in stages and "format_repair" in stages
|
||||
# Exactly one repair (not a retry loop).
|
||||
assert stages.count("format_repair") == 1
|
||||
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
seat_a_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "seat-a"}
|
||||
assert seat_a_stages == {"expert", "format_repair"}
|
||||
|
||||
|
||||
# ── 3. cross_review + cross_review_repair coexist ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_repair_coexist(svc):
|
||||
"""An invalid cross-review output triggers exactly one ``cross_review_repair``
|
||||
DeerFlow call; cross_review and cross_review_repair rows coexist for the
|
||||
manager."""
|
||||
adapter = _RecordingAdapter(svc)
|
||||
adapter.set_output(MANAGER, "cross_review", "totally not json")
|
||||
adapter.set_output(MANAGER, "cross_review_repair", json.dumps({
|
||||
"summary": "repaired review", "confirmed_candidates": [],
|
||||
}))
|
||||
|
||||
state = _state()
|
||||
# Pretend experts already ran so the cross-review barrier sees outcomes.
|
||||
state.seat_outcomes = {s: {"status": "completed"} for s in SEATS}
|
||||
state.current_round = 1
|
||||
|
||||
await CrossReviewExecutor(adapter, state).handle({}, _FakeCtx())
|
||||
|
||||
mgr_stages = [s for a, _, s in adapter.calls if a == MANAGER]
|
||||
assert "cross_review" in mgr_stages and "cross_review_repair" in mgr_stages
|
||||
assert mgr_stages.count("cross_review_repair") == 1
|
||||
|
||||
rows = await svc.store.seat_runs.get_by_maf_run("run-1")
|
||||
mgr_row_stages = {r["execution_stage"] for r in rows if r["agent_id"] == MANAGER}
|
||||
assert {"cross_review", "cross_review_repair"}.issubset(mgr_row_stages)
|
||||
|
||||
# The repaired review was applied (deterministic degradation did NOT happen).
|
||||
assert state.cross_review_result is not None
|
||||
assert state.cross_review_result.get("schema_status") == "repaired"
|
||||
assert state.cross_review_result.get("summary") == "repaired review"
|
||||
|
||||
|
||||
# ── 4. manager present in participants-style config ──────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_role_wired_through_state(svc):
|
||||
"""Sanity: RoundtableState carries manager_agent_id and the planner uses it
|
||||
for the planner seat-turn (not a seat id)."""
|
||||
state = _state()
|
||||
assert state.manager_agent_id == MANAGER
|
||||
assert MANAGER not in state.seat_agent_ids
|
||||
|
||||
adapter = _RecordingAdapter(svc)
|
||||
adapter.set_output(MANAGER, "planner", "plan")
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(adapter, state).handle("task", ctx)
|
||||
|
||||
planner_calls = [(a, s) for a, _, s in adapter.calls if s == "planner"]
|
||||
assert planner_calls == [(MANAGER, "planner")], \
|
||||
"planner stage must run as the MANAGER, not a seat"
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Phase-3 acceptance tests (P0-2): real seat-run identity & idempotency.
|
||||
|
||||
These tests use a REAL SQLite ``MafSeatRunRepository`` (no mocks on the
|
||||
repository) to exercise the Phase-3 four-column unique constraint
|
||||
``(maf_run_id, agent_id, turn_index, execution_stage)`` and the idempotent
|
||||
``get_or_create_seat_run`` acquisition.
|
||||
|
||||
They prove the core Phase-3 principle:
|
||||
|
||||
The business round ``turn_index`` never changes, but every distinct
|
||||
DeerFlow execution gets its own persistent identity via ``execution_stage``.
|
||||
|
||||
Coverage:
|
||||
1. manager planner → cross_review → finalizer within ONE business round
|
||||
coexist (same agent, same turn, different stage).
|
||||
2. expert → format_repair within one business round coexist (same agent,
|
||||
same turn, different stage).
|
||||
3. The same four-tuple second insert is rejected by the DB constraint.
|
||||
4. ``get_or_create_seat_run`` returns the existing row (created=False) instead
|
||||
of launching a duplicate when one already exists.
|
||||
5. Process recovery: an already-completed stage is NOT re-created.
|
||||
6. Idempotent Phase-3 migration + old rows back-filled to ``legacy``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import Base
|
||||
from deerflow.persistence.maf_roundtable.seat_runs import MafSeatRunRepository
|
||||
|
||||
# ── fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repo():
|
||||
"""A real SQLite-backed MafSeatRunRepository over the full schema."""
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
# Seed the parent run row so the FK on seat_runs is satisfiable.
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="s1", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="draft", write_policy="confirm_high_risk",
|
||||
max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="r1", session_id="s1", status="pending",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
r = MafSeatRunRepository(sf)
|
||||
yield r
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── 1. Same business round, different stages coexist (manager path) ─────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_stages_coexist_within_one_business_round(repo):
|
||||
"""planner / cross_review / finalizer all share (run, manager, turn) but
|
||||
differ by execution_stage — all three must insert without conflict."""
|
||||
common = dict(maf_run_id="r1", agent_id="manager-1", turn_index=1)
|
||||
await repo.create_seat_run(execution_stage="planner", **common)
|
||||
await repo.create_seat_run(execution_stage="cross_review", **common)
|
||||
await repo.create_seat_run(execution_stage="finalizer", **common)
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
stages = {r["execution_stage"] for r in rows}
|
||||
assert stages == {"planner", "cross_review", "finalizer"}
|
||||
# All share the SAME business turn_index (never mutated to dodge collision).
|
||||
assert {r["turn_index"] for r in rows} == {1}
|
||||
|
||||
|
||||
# ── 2. expert + format_repair coexist within one business round ─────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expert_and_format_repair_coexist(repo):
|
||||
"""An expert seat turn and its one-shot format-repair retry share the
|
||||
business round but are distinct DeerFlow executions."""
|
||||
common = dict(maf_run_id="r1", agent_id="expert-1", turn_index=1)
|
||||
await repo.create_seat_run(execution_stage="expert", **common)
|
||||
# Same agent, same turn, DIFFERENT stage — must NOT conflict.
|
||||
await repo.create_seat_run(execution_stage="format_repair", **common)
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
assert {r["execution_stage"] for r in rows} == {"expert", "format_repair"}
|
||||
|
||||
|
||||
# ── 3. Same four-tuple second insert is rejected ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_four_tuple_rejected(repo):
|
||||
"""Inserting the exact same (run, agent, turn, stage) twice must raise —
|
||||
the four-column unique constraint enforces at-most-one DeerFlow execution
|
||||
per stage per business round."""
|
||||
common = dict(maf_run_id="r1", agent_id="a1", turn_index=2, execution_stage="expert")
|
||||
await repo.create_seat_run(**common)
|
||||
with pytest.raises(IntegrityError):
|
||||
await repo.create_seat_run(**common)
|
||||
|
||||
|
||||
# ── 4. get_or_create_seat_run is idempotent ─────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_returns_existing_not_duplicate(repo):
|
||||
"""When a seat-run for the four-tuple already exists, get_or_create must
|
||||
return it with created=False rather than launching a duplicate."""
|
||||
key = dict(maf_run_id="r1", agent_id="a2", turn_index=3, execution_stage="cross_review")
|
||||
row1, created1 = await repo.get_or_create_seat_run(**key)
|
||||
assert created1 is True
|
||||
assert row1["execution_stage"] == "cross_review"
|
||||
|
||||
# Second acquisition — existing row returned, NO duplicate created.
|
||||
row2, created2 = await repo.get_or_create_seat_run(**key)
|
||||
assert created2 is False
|
||||
assert row2["id"] == row1["id"]
|
||||
|
||||
rows = await repo.get_by_maf_run("r1")
|
||||
cross_review_rows = [r for r in rows if r["agent_id"] == "a2"]
|
||||
assert len(cross_review_rows) == 1 # exactly one, no duplicate
|
||||
|
||||
|
||||
# ── 5. Process recovery: completed stage is NOT re-created ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recovery_completed_stage_not_recreated(repo):
|
||||
"""After a simulated restart, acquiring the same four-tuple returns the
|
||||
already-persisted row (created=False). The caller must replay the
|
||||
checkpoint's saved stage result instead of starting a new DeerFlow run."""
|
||||
key = dict(maf_run_id="r1", agent_id="a3", turn_index=4, execution_stage="finalizer")
|
||||
row, created = await repo.get_or_create_seat_run(**key)
|
||||
assert created is True
|
||||
await repo.mark_completed(row["id"], checkpoint_id="ckpt-xyz")
|
||||
|
||||
# Simulated restart: fresh acquisition of the same stage.
|
||||
row2, created2 = await repo.get_or_create_seat_run(**key)
|
||||
assert created2 is False
|
||||
assert row2["id"] == row["id"]
|
||||
assert row2["status"] == "completed"
|
||||
assert row2["workflow_checkpoint_id"] == "ckpt-xyz"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_by_four_tuple_lookup(repo):
|
||||
"""Direct four-tuple lookup returns the row or None."""
|
||||
key = dict(maf_run_id="r1", agent_id="a4", turn_index=5, execution_stage="expert")
|
||||
assert await repo.get_by_four_tuple(**key) is None
|
||||
await repo.create_seat_run(**key)
|
||||
found = await repo.get_by_four_tuple(**key)
|
||||
assert found is not None
|
||||
assert found["agent_id"] == "a4"
|
||||
assert found["execution_stage"] == "expert"
|
||||
|
||||
|
||||
# ── 6. Phase-3 migration: idempotent + back-fills legacy ────────────────────
|
||||
|
||||
|
||||
def _load_phase3_migration():
|
||||
here = Path(__file__).resolve().parent.parent
|
||||
script = here / "scripts" / "migrate_maf_roundtable_phase3.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_phase3", script)
|
||||
assert spec and spec.loader, "could not load phase3 migration script"
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase3_migration_idempotent_and_backfills_legacy():
|
||||
"""Run the Phase-3 migration against a Phase-2 schema (old 3-tuple
|
||||
constraint), then re-run it (idempotent). Old seat-run rows are
|
||||
back-filled with execution_stage='legacy'."""
|
||||
mod = _load_phase3_migration()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
|
||||
# Build the OLD Phase-2 seat_runs table (3-tuple unique, NO execution_stage).
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
session_id VARCHAR(64) NOT NULL,
|
||||
status VARCHAR(32) NOT NULL,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_sessions (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
user_id VARCHAR(64) NOT NULL,
|
||||
title VARCHAR(255),
|
||||
task_text TEXT,
|
||||
mode VARCHAR(32),
|
||||
status VARCHAR(32),
|
||||
write_policy VARCHAR(32),
|
||||
max_rounds INTEGER,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text("""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE
|
||||
)
|
||||
"""))
|
||||
await conn.execute(text(
|
||||
"CREATE UNIQUE INDEX uq_maf_seat_runs_run_agent_turn "
|
||||
"ON maf_roundtable_seat_runs (maf_run_id, agent_id, turn_index)"
|
||||
))
|
||||
# Seed parent run + a pre-Phase-3 seat-run row.
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_sessions (id, user_id, title, task_text, mode, status, write_policy, max_rounds, created_at, updated_at) "
|
||||
"VALUES ('s1','u1','t','t','structured','draft','confirm_high_risk',8,'2026-01-01','2026-01-01')"
|
||||
))
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_runs (id, session_id, status, created_at, updated_at) "
|
||||
"VALUES ('r1','s1','pending','2026-01-01','2026-01-01')"
|
||||
))
|
||||
await conn.execute(text(
|
||||
"INSERT INTO maf_roundtable_seat_runs (id, maf_run_id, agent_id, turn_index, status, created_at, updated_at) "
|
||||
"VALUES ('sr1','r1','a1',1,'completed','2026-01-01','2026-01-01')"
|
||||
))
|
||||
|
||||
# Run the migration.
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
|
||||
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
def _verify(sync_conn):
|
||||
insp = sa_inspect(sync_conn)
|
||||
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_runs")}
|
||||
assert "execution_stage" in cols
|
||||
# The NEW four-column unique constraint/index is present.
|
||||
new_constraint = mod._find_unique_constraint(
|
||||
insp, "maf_roundtable_seat_runs",
|
||||
expected_cols={"maf_run_id", "agent_id", "turn_index", "execution_stage"},
|
||||
)
|
||||
assert new_constraint is not None, "new four-column unique constraint missing"
|
||||
# The OLD three-column constraint must be gone (rebuilt in SQLite).
|
||||
old_constraint = mod._find_unique_constraint(
|
||||
insp, "maf_roundtable_seat_runs",
|
||||
expected_cols={"maf_run_id", "agent_id", "turn_index"},
|
||||
)
|
||||
assert old_constraint is None, "old three-column constraint still present after migration"
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(_verify)
|
||||
|
||||
# Old row back-filled to 'legacy'.
|
||||
async with sf() as session:
|
||||
row = (await session.execute(
|
||||
text("SELECT execution_stage FROM maf_roundtable_seat_runs WHERE id='sr1'")
|
||||
)).one_or_none()
|
||||
assert row is not None
|
||||
assert row[0] == "legacy"
|
||||
|
||||
# Re-run the migration — must be idempotent (no error).
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=[0])
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(mod._do_migrate_sync, dry_run=True, logger=mod.logger, changes_ref=[0])
|
||||
|
||||
# After migration, the store can create rows using the new column, and the
|
||||
# constraint allows cross_review + finalizer to coexist for the same
|
||||
# business round (the regression the migration fixes).
|
||||
store = make_maf_roundtable_store(sf)
|
||||
await store.seat_runs.create_seat_run(
|
||||
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="cross_review"
|
||||
)
|
||||
await store.seat_runs.create_seat_run(
|
||||
maf_run_id="r1", agent_id="mgr", turn_index=1, execution_stage="finalizer"
|
||||
)
|
||||
rows = await store.seat_runs.get_by_maf_run("r1")
|
||||
new_mgr_stages = {r["execution_stage"] for r in rows if r["agent_id"] == "mgr"}
|
||||
assert new_mgr_stages == {"cross_review", "finalizer"}
|
||||
|
||||
await engine.dispose()
|
||||
@@ -22,6 +22,7 @@ Additionally covers:
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
@@ -869,8 +870,9 @@ async def test_audit_event_failure_prevents_external_write(store):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deerflow_task_error_marks_run_failed(store):
|
||||
"""P1: when the DeerFlow agent run raises, the MAF run is marked failed
|
||||
(not completed), with a sanitized error message."""
|
||||
"""P3: when a DeerFlow seat turn raises, the seat is soft-failed
|
||||
(``partial_failure``) and the structured run still completes degraded;
|
||||
secrets must not appear in persisted seat / run errors."""
|
||||
svc = _svc(store)
|
||||
s = await svc.create_session(
|
||||
user_id="u1",
|
||||
@@ -890,9 +892,20 @@ async def test_deerflow_task_error_marks_run_failed(store):
|
||||
await runner.execute(s, run)
|
||||
|
||||
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
|
||||
assert final_run["status"] == "failed"
|
||||
assert final_run["status"] == "completed"
|
||||
assert final_run["finished_at"] is not None
|
||||
assert "sk-secret-12345" not in (final_run.get("error_message_safe") or "")
|
||||
ckpt = final_run.get("maf_checkpoint") or {}
|
||||
assert ckpt.get("partial_failure") is True
|
||||
outcome = (ckpt.get("seat_outcomes") or {}).get("boom-seat") or {}
|
||||
assert outcome.get("status") in ("failed", "error")
|
||||
persisted = " ".join(
|
||||
[
|
||||
str(final_run.get("error_message_safe") or ""),
|
||||
str(outcome.get("error") or ""),
|
||||
str(ckpt),
|
||||
]
|
||||
)
|
||||
assert "sk-secret-12345" not in persisted
|
||||
|
||||
|
||||
# ── 13. Secrets never in events / errors / logs ──────────────────────────
|
||||
@@ -1200,3 +1213,107 @@ async def test_capability_snapshot_drives_seat_config(store):
|
||||
|
||||
caps3 = _agent_capabilities_from_snapshot(None, "seat-1")
|
||||
assert caps3 == {}
|
||||
|
||||
|
||||
# ── 22. P1-3: Real WorkflowBuilder manager stages via WorkflowRunner ─────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_workflow_manager_stages_persisted(store):
|
||||
"""P1-3: drive a REAL MAF WorkflowRunner (real WorkflowBuilder, real SQLite
|
||||
repository, real adapter + seat-run store) through a structured run with a
|
||||
manager + 2 seats. Only the runtime client is faked (no real LLM).
|
||||
|
||||
Asserts:
|
||||
- manager has planner, cross_review, finalizer seat-run rows
|
||||
- cross_review.turn_index == finalizer.turn_index
|
||||
- cross_review.execution_stage != finalizer.execution_stage
|
||||
"""
|
||||
svc = _svc(store)
|
||||
s = await svc.create_session(
|
||||
user_id="u1",
|
||||
body={
|
||||
"task_text": "分析市场趋势",
|
||||
"mode": "structured",
|
||||
"manager_agent_id": "manager-1",
|
||||
"participants": [
|
||||
{"agent_id": "manager-1", "role": "manager"},
|
||||
{"agent_id": "seat-a", "role": "seat"},
|
||||
{"agent_id": "seat-b", "role": "seat"},
|
||||
],
|
||||
},
|
||||
)
|
||||
run = await svc.create_run(session_id=s["id"], user_id="u1")
|
||||
|
||||
app = _make_app(store)
|
||||
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
||||
|
||||
manager_responses = iter([
|
||||
"分工:seat-a 负责需求分析,seat-b 负责竞品分析",
|
||||
json.dumps({
|
||||
"summary": "审查通过,证据充分",
|
||||
"evidence_ok": True,
|
||||
"confirmed_candidates": [],
|
||||
}),
|
||||
"最终结论:市场看好,建议推进",
|
||||
])
|
||||
seat_responses = {
|
||||
"seat-a": json.dumps({"summary": "需求旺盛", "findings": ["增长10%"]}),
|
||||
"seat-b": json.dumps({"summary": "竞品A领先", "findings": ["份额30%"]}),
|
||||
}
|
||||
|
||||
call_log: list[dict] = []
|
||||
thread_counter = itertools.count()
|
||||
created_threads: list[str] = []
|
||||
latest_agent_by_thread: dict[str, str] = {}
|
||||
|
||||
async def fake_create_thread(principal):
|
||||
thread_id = f"thread-{principal.session_id[:8]}-{next(thread_counter)}"
|
||||
created_threads.append(thread_id)
|
||||
return thread_id
|
||||
|
||||
async def fake_start_agent_run(principal, *, thread_id, agent_id, message, **kwargs):
|
||||
latest_agent_by_thread[thread_id] = agent_id
|
||||
call_log.append({"agent_id": agent_id, "thread_id": thread_id, "message": message})
|
||||
return _FakeRunRecord(run_id=f"df-{agent_id}-{len(call_log)}")
|
||||
|
||||
async def fake_read_messages(principal, thread_id):
|
||||
agent_id = latest_agent_by_thread[thread_id]
|
||||
if agent_id == "manager-1":
|
||||
content = next(manager_responses)
|
||||
else:
|
||||
content = seat_responses[agent_id]
|
||||
return [{"type": "ai", "content": content, "usage_metadata": {"input_tokens": 5, "output_tokens": 5}}]
|
||||
|
||||
runner.adapter._client.create_thread = fake_create_thread
|
||||
runner.adapter._client.start_agent_run = fake_start_agent_run
|
||||
runner.adapter._client.read_thread_messages = fake_read_messages
|
||||
runner.adapter._client.cancel_run = AsyncMock(return_value=True)
|
||||
|
||||
await runner.execute(s, run)
|
||||
|
||||
final_run = await svc.get_run(run_id=run["id"], user_id="u1")
|
||||
assert final_run["status"] == "completed"
|
||||
|
||||
seat_runs = await store.seat_runs.get_by_maf_run(run["id"])
|
||||
mgr_rows = [r for r in seat_runs if r["agent_id"] == "manager-1"]
|
||||
mgr_stages = {r["execution_stage"] for r in mgr_rows}
|
||||
assert {"planner", "cross_review", "finalizer"}.issubset(mgr_stages), \
|
||||
f"manager stages missing; got {mgr_stages}"
|
||||
|
||||
cr_rows = [r for r in mgr_rows if r["execution_stage"] == "cross_review"]
|
||||
fin_rows = [r for r in mgr_rows if r["execution_stage"] == "finalizer"]
|
||||
assert len(cr_rows) == 1 and len(fin_rows) == 1
|
||||
assert cr_rows[0]["turn_index"] == fin_rows[0]["turn_index"]
|
||||
assert cr_rows[0]["execution_stage"] != fin_rows[0]["execution_stage"]
|
||||
|
||||
assert len(created_threads) == 3
|
||||
assert len(set(created_threads)) == 3
|
||||
|
||||
ckpt = final_run["maf_checkpoint"]
|
||||
assert "最终结论:市场看好" in str(ckpt)
|
||||
assert any(
|
||||
r.get("agent_id") == "seat-a"
|
||||
and (r.get("structured_output") or {}).get("summary") == "需求旺盛"
|
||||
for r in ckpt["turn_results"]
|
||||
)
|
||||
@@ -474,3 +474,43 @@ async def test_identity_from_request_state_not_body(app_and_client):
|
||||
headers={"X-Test-User": "bob"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ── Phase 3 resume / recover ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_and_recover_conflict_when_not_eligible(app_and_client):
|
||||
"""Resume requires waiting_approval; recover requires running|waiting_approval."""
|
||||
app, client, _ = app_and_client
|
||||
app.state.maf_auto_dispatch = False
|
||||
|
||||
resp = await client.post(
|
||||
"/api/maf-roundtable/sessions",
|
||||
json={"task_text": "t"},
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
session_id = resp.json()["id"]
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/sessions/{session_id}/runs",
|
||||
json={},
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
run_id = resp.json()["id"]
|
||||
# Pending runs (auto-dispatch off) are not eligible for resume/recover.
|
||||
assert resp.json()["status"] == "pending"
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/runs/{run_id}/resume",
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/maf-roundtable/runs/{run_id}/recover",
|
||||
headers={"X-Test-User": "alice"},
|
||||
)
|
||||
assert resp.status_code == 409
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Unit tests for seat structured output + one-shot repair."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.maf_roundtable.seat_output import (
|
||||
build_repair_prompt,
|
||||
parse_seat_output,
|
||||
validate_or_repair_seat_output,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_valid_seat_output():
|
||||
text = '{"summary":"ok","findings":["f1"],"proposed_actions":["do x"]}'
|
||||
model, errors = parse_seat_output(text)
|
||||
assert errors == []
|
||||
assert model is not None
|
||||
assert model.summary == "ok"
|
||||
assert model.findings == ["f1"]
|
||||
|
||||
|
||||
def test_parse_fenced_json():
|
||||
text = '说明\n```json\n{"summary":"s"}\n```\n'
|
||||
model, errors = parse_seat_output(text)
|
||||
assert model is not None
|
||||
assert errors == []
|
||||
|
||||
|
||||
def test_parse_invalid_missing_summary():
|
||||
model, errors = parse_seat_output('{"findings":[]}')
|
||||
assert model is None
|
||||
assert errors
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_ok_no_repair():
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content='{"summary":"fine"}',
|
||||
repair_runner=None,
|
||||
)
|
||||
assert result["schema_status"] == "ok"
|
||||
assert result["repair_attempted"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_shot_repair_success():
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _repair(prompt: str) -> str:
|
||||
calls["n"] += 1
|
||||
assert "SeatTurnOutput" in prompt or "校验" in prompt
|
||||
return '{"summary":"repaired","findings":["x"]}'
|
||||
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content="not json at all",
|
||||
repair_runner=_repair,
|
||||
)
|
||||
assert calls["n"] == 1
|
||||
assert result["schema_status"] == "repaired"
|
||||
assert result["output"]["summary"] == "repaired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repair_still_invalid_stops():
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _repair(prompt: str) -> str:
|
||||
calls["n"] += 1
|
||||
return "still not json"
|
||||
|
||||
result = await validate_or_repair_seat_output(
|
||||
raw_content="garbage",
|
||||
repair_runner=_repair,
|
||||
)
|
||||
assert calls["n"] == 1
|
||||
assert result["schema_status"] == "schema_invalid"
|
||||
assert result["repair_attempted"] is True
|
||||
|
||||
|
||||
def test_build_repair_prompt_includes_errors():
|
||||
prompt = build_repair_prompt("raw", ["summary: Field required"])
|
||||
assert "summary: Field required" in prompt
|
||||
assert "raw" in prompt
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Phase 3 structured workflow acceptance tests (MAF-03)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.fact_ledger import executed_action_ids, proposed_action_ids
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
CrossReviewExecutor,
|
||||
PlannerExecutor,
|
||||
RoundtableState,
|
||||
SeatExecutor,
|
||||
)
|
||||
from app.maf_roundtable.workflow_factory import create_workflow
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
|
||||
|
||||
class _FakeCtx:
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[Any] = []
|
||||
self.outputs: list[Any] = []
|
||||
|
||||
async def send_message(self, msg: Any) -> None:
|
||||
self.messages.append(msg)
|
||||
|
||||
async def yield_output(self, msg: Any) -> None:
|
||||
self.outputs.append(msg)
|
||||
|
||||
|
||||
class _FakeSeatResponse:
|
||||
def __init__(self, content: str = "", tool_calls: list | None = None, error: str | None = None) -> None:
|
||||
self.content = content
|
||||
self.tool_calls = tool_calls or []
|
||||
self.error = error
|
||||
self.usage = {"input_tokens": 1, "output_tokens": 1}
|
||||
|
||||
|
||||
def _state(seat_ids: list[str] | None = None) -> RoundtableState:
|
||||
seats = seat_ids or ["seat-a", "seat-b", "seat-c"]
|
||||
return RoundtableState(
|
||||
session_id="s1",
|
||||
run_id="r1",
|
||||
task_text="分析市场",
|
||||
mode="structured",
|
||||
model_name=None,
|
||||
reasoning_effort=None,
|
||||
max_rounds=8,
|
||||
write_policy="confirm_high_risk",
|
||||
manager_agent_id=None,
|
||||
seat_agent_ids=seats,
|
||||
thread_map={a: f"t-{a}" for a in seats},
|
||||
capability_snapshot=None,
|
||||
seat_timeout_seconds=5,
|
||||
schema_repair_enabled=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_deterministic_assignments():
|
||||
state = _state()
|
||||
adapter = SimpleNamespace(_emit_critical=AsyncMock())
|
||||
planner = PlannerExecutor(adapter, state)
|
||||
ctx = _FakeCtx()
|
||||
await planner.handle("task", ctx)
|
||||
assert "planner" in state.completed_nodes
|
||||
assert state.plan is not None
|
||||
assert set(state.plan["assignments"]) == {"seat-a", "seat-b", "seat-c"}
|
||||
assert ctx.messages[-1]["decision"] == "parallel_seats"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_skips_when_completed():
|
||||
state = _state()
|
||||
state.mark_completed("planner")
|
||||
state.plan = {"source": "deterministic", "assignments": {}}
|
||||
adapter = SimpleNamespace(run_seat_turn=AsyncMock())
|
||||
planner = PlannerExecutor(adapter, state)
|
||||
ctx = _FakeCtx()
|
||||
await planner.handle("task", ctx)
|
||||
adapter.run_seat_turn.assert_not_called()
|
||||
assert ctx.messages[-1].get("replayed") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seat_partial_failure_does_not_raise():
|
||||
state = _state(["seat-a", "seat-b"])
|
||||
state.plan = {
|
||||
"assignments": {
|
||||
"seat-a": {"instruction": "do a", "role": "A"},
|
||||
"seat-b": {"instruction": "do b", "role": "B"},
|
||||
}
|
||||
}
|
||||
state.current_round = 1
|
||||
|
||||
async def _run(**kwargs):
|
||||
if kwargs["agent_id"] == "seat-a":
|
||||
raise RuntimeError("boom")
|
||||
return _FakeSeatResponse(content='{"summary":"ok-b","findings":["f"]}')
|
||||
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(side_effect=_run),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
|
||||
seat_a = SeatExecutor(adapter, state, "seat-a")
|
||||
seat_b = SeatExecutor(adapter, state, "seat-b")
|
||||
ctx_a, ctx_b = _FakeCtx(), _FakeCtx()
|
||||
await seat_a.handle({}, ctx_a)
|
||||
await seat_b.handle({}, ctx_b)
|
||||
|
||||
assert state.partial_failure is True
|
||||
assert state.seat_outcomes["seat-a"]["status"] in ("failed", "error")
|
||||
assert state.seat_outcomes["seat-b"]["status"] == "completed"
|
||||
assert any(r.get("agent_id") == "seat-b" and r.get("content") for r in state.turn_results)
|
||||
assert "seat_seat-a" in state.completed_nodes
|
||||
assert "seat_seat-b" in state.completed_nodes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seat_schema_one_repair_then_invalid():
|
||||
state = _state(["seat-a"])
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
state.current_round = 1
|
||||
calls = {"n": 0}
|
||||
|
||||
async def _run(**kwargs):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
return _FakeSeatResponse(content="not-json")
|
||||
return _FakeSeatResponse(content="still-bad")
|
||||
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(side_effect=_run),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
seat = SeatExecutor(adapter, state, "seat-a")
|
||||
await seat.handle({}, _FakeCtx())
|
||||
# 1 expert + 1 repair
|
||||
assert calls["n"] == 2
|
||||
row = state.turn_results[-1]
|
||||
assert row["schema_status"] == "schema_invalid"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proposed_not_executed_in_ledger():
|
||||
state = _state(["seat-a"])
|
||||
state.plan = {"assignments": {"seat-a": {"instruction": "x", "role": "A"}}}
|
||||
state.current_round = 1
|
||||
content = (
|
||||
'{"summary":"s","findings":["f"],'
|
||||
'"proposed_actions":[{"action_id":"p1","text":"write"}],'
|
||||
'"executed_actions":[{"action_id":"forge","text":"lie"}]}'
|
||||
)
|
||||
adapter = SimpleNamespace(
|
||||
run_seat_turn=AsyncMock(return_value=_FakeSeatResponse(content=content)),
|
||||
_emit_critical=AsyncMock(),
|
||||
_service=SimpleNamespace(store=SimpleNamespace(actions=SimpleNamespace(list_actions=AsyncMock(return_value=[])))),
|
||||
)
|
||||
await SeatExecutor(adapter, state, "seat-a").handle({}, _FakeCtx())
|
||||
assert "p1" in proposed_action_ids(state.fact_ledger)
|
||||
assert "forge" not in executed_action_ids(state.fact_ledger)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_review_barrier_marks_missing():
|
||||
state = _state(["seat-a", "seat-b"])
|
||||
state.seat_outcomes = {"seat-a": {"status": "completed"}}
|
||||
state.current_round = 1
|
||||
adapter = SimpleNamespace(run_seat_turn=AsyncMock(), _emit_critical=AsyncMock())
|
||||
review = CrossReviewExecutor(adapter, state)
|
||||
await review.handle({}, _FakeCtx())
|
||||
assert state.seat_outcomes["seat-b"]["status"] == "missing"
|
||||
assert state.partial_failure is True
|
||||
assert "cross_review" in state.completed_nodes
|
||||
|
||||
|
||||
def test_structured_factory_requires_planner():
|
||||
with pytest.raises(ValueError):
|
||||
create_workflow(
|
||||
mode="structured",
|
||||
seat_executors=[],
|
||||
approval_gate=object(),
|
||||
finalizer=object(),
|
||||
waiting_terminal=object(),
|
||||
seat_agent_ids=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_skips_completed_seats(tmp_path, monkeypatch):
|
||||
"""completed_nodes must not re-invoke run_seat_turn on recover."""
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'p3r.db'}")
|
||||
async with engine.begin() as conn:
|
||||
import deerflow.persistence.maf_roundtable.model # noqa: F401
|
||||
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
svc = make_maf_service(store)
|
||||
session = await svc.create_session(
|
||||
user_id="u1",
|
||||
body={
|
||||
"task_text": "t",
|
||||
"mode": "structured",
|
||||
"participants": [
|
||||
{"agent_id": "seat-a", "role": "seat"},
|
||||
{"agent_id": "seat-b", "role": "seat"},
|
||||
],
|
||||
},
|
||||
)
|
||||
run = await svc.create_run(session_id=session["id"], user_id="u1")
|
||||
|
||||
# Pretend planner + seat-a already finished; seat-b not yet.
|
||||
state = RoundtableState(
|
||||
session_id=session["id"],
|
||||
run_id=run["id"],
|
||||
task_text="t",
|
||||
mode="structured",
|
||||
model_name=None,
|
||||
reasoning_effort=None,
|
||||
max_rounds=8,
|
||||
write_policy="confirm_high_risk",
|
||||
manager_agent_id=None,
|
||||
seat_agent_ids=["seat-a", "seat-b"],
|
||||
thread_map={"seat-a": "ta", "seat-b": "tb"},
|
||||
capability_snapshot=session.get("capability_snapshot"),
|
||||
seat_timeout_seconds=30,
|
||||
schema_repair_enabled=False,
|
||||
)
|
||||
state.plan = {
|
||||
"source": "deterministic",
|
||||
"assignments": {
|
||||
"seat-a": {"instruction": "a", "role": "A"},
|
||||
"seat-b": {"instruction": "b", "role": "B"},
|
||||
},
|
||||
}
|
||||
state.current_round = 1
|
||||
state.stage = "experts"
|
||||
state.mark_completed("planner")
|
||||
state.mark_completed("seat_seat-a")
|
||||
state.seat_outcomes["seat-a"] = {"status": "completed"}
|
||||
state.turn_results.append({
|
||||
"agent_id": "seat-a",
|
||||
"turn_index": 1,
|
||||
"content": '{"summary":"a"}',
|
||||
"tool_calls": [],
|
||||
"error": None,
|
||||
})
|
||||
await store.runs.update_run(run["id"], {"status": "running", "maf_checkpoint": state.to_dict()})
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
async def _run_seat(**kwargs):
|
||||
aid = kwargs["agent_id"]
|
||||
calls.append(aid)
|
||||
return _FakeSeatResponse(content=f'{{"summary":"{aid}"}}')
|
||||
|
||||
app = SimpleNamespace(state=SimpleNamespace(maf_roundtable_service=svc, checkpointer=None))
|
||||
runner = WorkflowRunner(app, service=svc, user_id="u1")
|
||||
monkeypatch.setattr(runner._adapter, "run_seat_turn", AsyncMock(side_effect=_run_seat))
|
||||
monkeypatch.setattr(runner._adapter, "ensure_threads", AsyncMock(return_value=state.thread_map))
|
||||
monkeypatch.setattr(runner._adapter, "_emit_critical", AsyncMock())
|
||||
|
||||
# Avoid needing a real SDK stream — drive recover path by patching _run_workflow
|
||||
# to invoke remaining executors with skip semantics.
|
||||
async def _fake_run_workflow(workflow, st):
|
||||
# Simulate graph: planner (skip) → seats → cross_review → approval → finalizer
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
ApprovalGateExecutor,
|
||||
CrossReviewExecutor,
|
||||
FinalizerExecutor,
|
||||
PlannerExecutor,
|
||||
SeatExecutor,
|
||||
)
|
||||
|
||||
ctx = _FakeCtx()
|
||||
await PlannerExecutor(runner._adapter, st).handle("t", ctx)
|
||||
for aid in st.seat_agent_ids:
|
||||
await SeatExecutor(runner._adapter, st, aid).handle({}, _FakeCtx())
|
||||
await CrossReviewExecutor(runner._adapter, st).handle({}, _FakeCtx())
|
||||
await ApprovalGateExecutor(runner._adapter, st, svc).handle({}, _FakeCtx())
|
||||
await FinalizerExecutor(runner._adapter, st).handle({}, _FakeCtx())
|
||||
st.status = "completed"
|
||||
|
||||
monkeypatch.setattr(runner, "_run_workflow", _fake_run_workflow)
|
||||
try:
|
||||
await runner.recover(run["id"])
|
||||
# seat-a was completed — must not be called; seat-b should run once.
|
||||
assert "seat-a" not in calls
|
||||
assert calls.count("seat-b") == 1
|
||||
finally:
|
||||
await engine.dispose()
|
||||
Reference in new issue
Block a user