fix(maf): harden stage-4 quality gate (P0/P1)
P0-1 migration: detect the same-named stage-3 execution_stage CHECK by sqltext (must contain rework + manager_review), SQLite rebuild / PG-MySQL drop+add, PG boolean default TRUE, validate_schema checks constraint content + seat_tasks status CHECK/unique/FKs; tests start from a real stage-3 schema. P0-2: a missing/invalid manager review (timeout/exception/empty/invalid JSON/missing per-seat decision) is never an acceptance - system rework under the cap, contract fallback (reassign/controlled_gap/ask_user) afterwards, source=system events. P0-3: unaccepted deliveries are isolated from the fact ledger, cross-review and final report (deferred merge, delivery_json envelope, accepted_task_results in checkpoint, accepted-only synthesis, no turn_results fallback). P0-4: seat-task init is atomic get-or-create over the full expected set; recovery fills missing tasks without overwriting frozen contracts; quality gate blocks missing required tasks. P1: zero config values honoured; duplicate detection persists SHA-256 conclusion digests in history (survives restarts); findings never satisfy evidence, evidence_refs must resolve to trusted ledger refs (unverified_evidence_ref); conversation SSE projection tests added and manager/finalizer messages are no longer mislabelled as seat.delivery.submitted.
This commit is contained in:
1 parent
a786cdcad0
commit
1aeee86984
24 files changed
+4706
-47
No files matched your search
@@ -875,8 +875,9 @@ async def test_sse_generator_realtime_terminal_stops(svc, store, hub):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p0_1_atomic_projection_both_events_or_neither(svc, store):
|
||||
"""agent.message.completed must atomically project both
|
||||
agent.message.complete AND seat.delivery.submitted."""
|
||||
"""A genuine seat delivery (agent.message.completed with a seat delivery
|
||||
execution_stage) must atomically project both agent.message.complete AND
|
||||
seat.delivery.submitted."""
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -914,7 +915,11 @@ async def test_p0_1_atomic_projection_both_events_or_neither(svc, store):
|
||||
"event_type": "agent.message.completed",
|
||||
"agent_id": "seat_a",
|
||||
"turn_index": 1,
|
||||
"payload": {"content": "result", "schema_status": "valid"},
|
||||
"payload": {
|
||||
"content": "result",
|
||||
"schema_status": "valid",
|
||||
"execution_stage": "expert",
|
||||
},
|
||||
}
|
||||
|
||||
written = await svc.bridge_workflow_event(
|
||||
@@ -1020,7 +1025,7 @@ async def test_p0_1_projection_failure_does_not_duplicate_on_retry(svc, store):
|
||||
id="audit_retry_1", session_id=session.id, run_id=run.id,
|
||||
seq=1, event_type="agent.message.completed",
|
||||
agent_id="seat_a", turn_index=0,
|
||||
payload_json={"content": "done"},
|
||||
payload_json={"content": "done", "execution_stage": "expert"},
|
||||
))
|
||||
|
||||
await svc.bridge_workflow_event(
|
||||
@@ -1029,7 +1034,7 @@ async def test_p0_1_projection_failure_does_not_duplicate_on_retry(svc, store):
|
||||
"event_type": "agent.message.completed",
|
||||
"agent_id": "seat_a",
|
||||
"turn_index": 0,
|
||||
"payload": {"content": "done"},
|
||||
"payload": {"content": "done", "execution_stage": "expert"},
|
||||
},
|
||||
source_audit_event_id="audit_retry_1",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Acceptance tests for the Stage-4 migration script (R3-Q quality gate).
|
||||
|
||||
P0-1: the tests start from a REAL Stage-3 schema — the seat_runs table is
|
||||
created by hand with the Stage-3 ``execution_stage`` CHECK (same constraint
|
||||
NAME as Stage 4, but WITHOUT ``rework`` / ``manager_review``). ``Base.metadata
|
||||
.create_all()`` is never used to fake the Stage-3 seat_runs table, because the
|
||||
current ORM metadata already carries the widened Stage-4 CHECK.
|
||||
|
||||
Verified behaviour:
|
||||
|
||||
1. A same-named Stage-3 constraint is detected as NOT widened (sqltext must
|
||||
contain both new stages — the name alone never proves Stage 4).
|
||||
2. SQLite: the incomplete constraint triggers the table rebuild; the original
|
||||
seat-run rows survive; indexes are recreated; a second run is a no-op.
|
||||
3. After migration, inserting ``rework`` and ``manager_review`` succeeds while
|
||||
an unknown stage is still rejected by the CHECK.
|
||||
4. Two consecutive runs produce no errors and no duplicate tables/indexes.
|
||||
5. ``validate_schema`` checks constraint CONTENT plus the seat_tasks status
|
||||
CHECK, unique key and foreign keys.
|
||||
6. PostgreSQL / MySQL DDL generation is asserted without a live database
|
||||
(drop+recreate on an existing old constraint, plain add when absent, no-op
|
||||
when already Stage 4; PostgreSQL boolean default is ``TRUE`` not ``1``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
Base,
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSeatRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
STAGE3_STAGES = (
|
||||
"'legacy','planner','expert','format_repair',"
|
||||
"'cross_review','cross_review_repair','finalizer','group_chat'"
|
||||
)
|
||||
|
||||
# Real Stage-3 seat_runs DDL — identical shape to the Phase-3 output, with the
|
||||
# Stage-3 (unwidened) execution_stage CHECK.
|
||||
STAGE3_SEAT_RUNS_DDL = f"""
|
||||
CREATE TABLE maf_roundtable_seat_runs (
|
||||
id VARCHAR(64) NOT NULL PRIMARY KEY,
|
||||
maf_run_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(64) NOT NULL,
|
||||
turn_index INTEGER NOT NULL,
|
||||
execution_stage VARCHAR(32) NOT NULL DEFAULT 'legacy',
|
||||
deerflow_run_id VARCHAR(64),
|
||||
deerflow_thread_id VARCHAR(64),
|
||||
sandbox_id VARCHAR(64),
|
||||
workflow_checkpoint_id VARCHAR(128),
|
||||
status VARCHAR(32) DEFAULT 'pending' NOT NULL,
|
||||
result_snapshot TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
error TEXT,
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL,
|
||||
FOREIGN KEY (maf_run_id) REFERENCES maf_roundtable_runs (id) ON DELETE CASCADE,
|
||||
CONSTRAINT uq_maf_seat_runs_run_agent_turn_stage
|
||||
UNIQUE (maf_run_id, agent_id, turn_index, execution_stage),
|
||||
CONSTRAINT ck_maf_seat_runs_execution_stage
|
||||
CHECK (execution_stage IN ({STAGE3_STAGES}))
|
||||
)
|
||||
"""
|
||||
|
||||
STAGE3_SEAT_RUNS_INDEXES = [
|
||||
"CREATE INDEX ix_maf_seat_runs_maf_run_id ON maf_roundtable_seat_runs (maf_run_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_agent_id ON maf_roundtable_seat_runs (agent_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_deerflow_run_id ON maf_roundtable_seat_runs (deerflow_run_id)",
|
||||
"CREATE INDEX ix_maf_seat_runs_status ON maf_roundtable_seat_runs (status)",
|
||||
]
|
||||
|
||||
|
||||
def _load_migration_module():
|
||||
"""Load scripts/migrate_maf_roundtable_stage4.py as a module."""
|
||||
here = Path(__file__).resolve().parent.parent
|
||||
script = here / "scripts" / "migrate_maf_roundtable_stage4.py"
|
||||
spec = importlib.util.spec_from_file_location("migrate_maf_roundtable_stage4", script)
|
||||
assert spec and spec.loader, "could not load migration script"
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
async def _create_stage3_schema(engine) -> None:
|
||||
"""Create a genuine Stage-3 schema.
|
||||
|
||||
sessions / runs come from the ORM metadata (they did not change in Stage 4);
|
||||
seat_runs is created BY HAND with the Stage-3 CHECK; seat_tasks does not
|
||||
exist yet.
|
||||
"""
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
lambda sync_conn: Base.metadata.create_all(
|
||||
sync_conn,
|
||||
tables=[
|
||||
MafRoundtableSessionRow.__table__,
|
||||
MafRoundtableRunRow.__table__,
|
||||
],
|
||||
)
|
||||
)
|
||||
await conn.execute(text(STAGE3_SEAT_RUNS_DDL))
|
||||
for ddl in STAGE3_SEAT_RUNS_INDEXES:
|
||||
await conn.execute(text(ddl))
|
||||
|
||||
|
||||
async def _seed_stage3_data(engine) -> None:
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="s1", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="draft", write_policy="confirm_high_risk",
|
||||
max_rounds=8, created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="r1", session_id="s1", status="completed",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr1", maf_run_id="r1", agent_id="seat-a", turn_index=1,
|
||||
execution_stage="expert", status="completed",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr2", maf_run_id="r1", agent_id="manager-1", turn_index=1,
|
||||
execution_stage="cross_review", status="completed",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _run_migrate(engine, mod) -> int:
|
||||
changes_ref = [0]
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
mod._do_migrate_sync, dry_run=False, logger=mod.logger, changes_ref=changes_ref
|
||||
)
|
||||
return changes_ref[0]
|
||||
|
||||
|
||||
def _seat_runs_check_sql(insp) -> str | None:
|
||||
for ck in insp.get_check_constraints("maf_roundtable_seat_runs"):
|
||||
if ck.get("name") == "ck_maf_seat_runs_execution_stage":
|
||||
return str(ck.get("sqltext") or "")
|
||||
return None
|
||||
|
||||
|
||||
# ── detection: same-named Stage-3 constraint is NOT Stage 4 ────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage3_same_name_constraint_detected_as_not_stage4():
|
||||
mod = _load_migration_module()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
try:
|
||||
await _create_stage3_schema(engine)
|
||||
|
||||
def _detect(sync_conn):
|
||||
insp = sa_inspect(sync_conn)
|
||||
return (
|
||||
mod._execution_stage_check_sql(insp),
|
||||
mod._has_stage4_execution_stage_check(insp),
|
||||
)
|
||||
|
||||
async with engine.begin() as conn:
|
||||
sql, is_stage4 = await conn.run_sync(_detect)
|
||||
|
||||
# The Stage-3 constraint exists under the same name ...
|
||||
assert sql is not None
|
||||
# ... but it does NOT allow the Stage-4 stages.
|
||||
assert "rework" not in sql
|
||||
assert "manager_review" not in sql
|
||||
assert is_stage4 is False
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── SQLite: real Stage-3 → Stage-4, idempotent, data preserved ─────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_widens_real_stage3_and_is_idempotent():
|
||||
mod = _load_migration_module()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
try:
|
||||
await _create_stage3_schema(engine)
|
||||
await _seed_stage3_data(engine)
|
||||
|
||||
# Sanity: before migration, 'rework' is rejected by the Stage-3 CHECK.
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
with pytest.raises(Exception):
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr-pre", maf_run_id="r1", agent_id="seat-x", turn_index=9,
|
||||
execution_stage="rework", status="pending",
|
||||
created_at=datetime(2026, 1, 1), updated_at=datetime(2026, 1, 1),
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
# Run the migration twice — the second run must be a no-op.
|
||||
first_changes = await _run_migrate(engine, mod)
|
||||
assert first_changes > 0, "first run must apply changes on a Stage-3 schema"
|
||||
|
||||
def _index_names(sync_conn):
|
||||
return sorted(
|
||||
idx["name"]
|
||||
for idx in sa_inspect(sync_conn).get_indexes("maf_roundtable_seat_runs")
|
||||
)
|
||||
|
||||
async with engine.begin() as conn:
|
||||
indexes_after_first = await conn.run_sync(_index_names)
|
||||
|
||||
second_changes = await _run_migrate(engine, mod)
|
||||
assert second_changes == 0, "second run must be a no-op"
|
||||
|
||||
async with engine.begin() as conn:
|
||||
indexes_after_second = await conn.run_sync(_index_names)
|
||||
assert indexes_after_first == indexes_after_second, "no duplicate indexes"
|
||||
|
||||
def _check(sync_conn):
|
||||
insp = sa_inspect(sync_conn)
|
||||
tables = set(insp.get_table_names())
|
||||
# No leftover rebuild artefacts, exactly one seat_runs table.
|
||||
assert "maf_roundtable_seat_runs__stage4" not in tables
|
||||
assert "maf_roundtable_seat_tasks" in tables
|
||||
|
||||
# The widened CHECK contains BOTH new stages.
|
||||
sql = _seat_runs_check_sql(insp)
|
||||
assert sql is not None
|
||||
assert "rework" in sql
|
||||
assert "manager_review" in sql
|
||||
|
||||
# seat_tasks contract: columns, unique key, status CHECK, FKs.
|
||||
cols = {c["name"] for c in insp.get_columns("maf_roundtable_seat_tasks")}
|
||||
for required in (
|
||||
"id", "session_id", "run_id", "task_key", "agent_id",
|
||||
"original_agent_id", "status", "required", "contract_json",
|
||||
"delivery_json", "history_json", "gap_json",
|
||||
):
|
||||
assert required in cols, f"missing column {required}"
|
||||
uq = [
|
||||
set(u.get("column_names", []))
|
||||
for u in insp.get_unique_constraints("maf_roundtable_seat_tasks")
|
||||
]
|
||||
assert {"run_id", "task_key"} in uq
|
||||
|
||||
# Original Stage-3 rows survived the rebuild untouched.
|
||||
rows = sync_conn.execute(text(
|
||||
"SELECT id, execution_stage, status FROM maf_roundtable_seat_runs ORDER BY id"
|
||||
)).fetchall()
|
||||
assert rows == [
|
||||
("sr1", "expert", "completed"),
|
||||
("sr2", "cross_review", "completed"),
|
||||
]
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(_check)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migration_allows_stage4_stages_and_rejects_unknown():
|
||||
mod = _load_migration_module()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
try:
|
||||
await _create_stage3_schema(engine)
|
||||
await _seed_stage3_data(engine)
|
||||
await _run_migrate(engine, mod)
|
||||
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
now = datetime(2026, 1, 2)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr-rework", maf_run_id="r1", agent_id="seat-a", turn_index=2,
|
||||
execution_stage="rework", status="completed",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr-review", maf_run_id="r1", agent_id="manager-1", turn_index=2,
|
||||
execution_stage="manager_review", status="completed",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
|
||||
async with sf() as session:
|
||||
stages = (
|
||||
(await session.execute(text(
|
||||
"SELECT execution_stage FROM maf_roundtable_seat_runs ORDER BY id"
|
||||
))).scalars().all()
|
||||
)
|
||||
assert "rework" in stages
|
||||
assert "manager_review" in stages
|
||||
|
||||
# The CHECK is still enforced — an unknown stage must be rejected.
|
||||
with pytest.raises(Exception):
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSeatRunRow(
|
||||
id="sr-bogus", maf_run_id="r1", agent_id="seat-a", turn_index=3,
|
||||
execution_stage="bogus_stage", status="pending",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── validate_schema: content checks, not just names ────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_schema_flags_stage3_then_passes_stage4():
|
||||
mod = _load_migration_module()
|
||||
engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||
try:
|
||||
await _create_stage3_schema(engine)
|
||||
await _seed_stage3_data(engine)
|
||||
|
||||
def _validate(sync_conn):
|
||||
return mod.validate_schema(sa_inspect(sync_conn))
|
||||
|
||||
# Before migration: seat_tasks missing AND the CHECK not widened.
|
||||
async with engine.begin() as conn:
|
||||
issues = await conn.run_sync(_validate)
|
||||
assert any("missing table" in i for i in issues)
|
||||
assert any("not widened" in i for i in issues)
|
||||
|
||||
await _run_migrate(engine, mod)
|
||||
async with engine.begin() as conn:
|
||||
issues = await conn.run_sync(_validate)
|
||||
assert issues == []
|
||||
|
||||
# Dropping the status CHECK must be reported (content validation).
|
||||
# SQLite cannot DROP CONSTRAINT, so simulate by checking a tampered
|
||||
# inspector view: rebuild without the constraint is out of scope here;
|
||||
# instead verify the FK/unique violations are reported when the table
|
||||
# is dropped entirely.
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("DROP TABLE maf_roundtable_seat_tasks"))
|
||||
async with engine.begin() as conn:
|
||||
issues = await conn.run_sync(_validate)
|
||||
assert any("missing table" in i for i in issues)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
# ── PostgreSQL / MySQL DDL generation (no live database) ───────────────────
|
||||
|
||||
|
||||
def test_widen_check_ddls_postgresql():
|
||||
mod = _load_migration_module()
|
||||
# Old same-named constraint present → drop then re-add.
|
||||
ddls = mod._widen_check_ddls("postgresql", check_exists=True, is_stage4=False)
|
||||
assert len(ddls) == 2
|
||||
assert "DROP CONSTRAINT ck_maf_seat_runs_execution_stage" in ddls[0]
|
||||
assert "ADD CONSTRAINT ck_maf_seat_runs_execution_stage" in ddls[1]
|
||||
for stage in ("rework", "manager_review"):
|
||||
assert stage in ddls[1]
|
||||
# No constraint at all → plain add.
|
||||
ddls = mod._widen_check_ddls("postgresql", check_exists=False, is_stage4=False)
|
||||
assert len(ddls) == 1
|
||||
assert ddls[0].startswith("ALTER TABLE")
|
||||
assert "DROP" not in ddls[0]
|
||||
# Already Stage 4 → no-op.
|
||||
assert mod._widen_check_ddls("postgresql", check_exists=True, is_stage4=True) == []
|
||||
|
||||
|
||||
def test_widen_check_ddls_mysql():
|
||||
mod = _load_migration_module()
|
||||
ddls = mod._widen_check_ddls("mysql", check_exists=True, is_stage4=False)
|
||||
assert len(ddls) == 2
|
||||
assert "DROP CHECK ck_maf_seat_runs_execution_stage" in ddls[0]
|
||||
assert "ADD CONSTRAINT ck_maf_seat_runs_execution_stage" in ddls[1]
|
||||
assert mod._widen_check_ddls("mysql", check_exists=True, is_stage4=True) == []
|
||||
|
||||
|
||||
def test_widen_check_ddls_sqlite_rebuilds():
|
||||
mod = _load_migration_module()
|
||||
ddls = mod._widen_check_ddls("sqlite", check_exists=True, is_stage4=False)
|
||||
joined = ";\n".join(ddls)
|
||||
assert "CREATE TABLE maf_roundtable_seat_runs__stage4" in joined
|
||||
assert "INSERT INTO maf_roundtable_seat_runs__stage4" in joined
|
||||
assert "DROP TABLE maf_roundtable_seat_runs" in joined
|
||||
assert "RENAME TO maf_roundtable_seat_runs" in joined
|
||||
assert "CREATE INDEX" in joined
|
||||
assert mod._widen_check_ddls("sqlite", check_exists=True, is_stage4=True) == []
|
||||
|
||||
|
||||
def test_seat_tasks_boolean_default_per_dialect():
|
||||
mod = _load_migration_module()
|
||||
pg = mod._seat_tasks_ddl("postgresql")
|
||||
assert "required BOOLEAN NOT NULL DEFAULT TRUE" in pg
|
||||
assert "DEFAULT 1" not in pg
|
||||
sqlite = mod._seat_tasks_ddl("sqlite")
|
||||
assert "required BOOLEAN NOT NULL DEFAULT 1" in sqlite
|
||||
mysql = mod._seat_tasks_ddl("mysql")
|
||||
assert "required BOOLEAN NOT NULL DEFAULT 1" in mysql
|
||||
|
||||
|
||||
def test_fake_inspector_detection_matrix():
|
||||
mod = _load_migration_module()
|
||||
|
||||
class FakeInspector:
|
||||
def __init__(self, checks):
|
||||
self._checks = checks
|
||||
|
||||
def get_check_constraints(self, table):
|
||||
return self._checks.get(table, [])
|
||||
|
||||
old = FakeInspector({
|
||||
"maf_roundtable_seat_runs": [
|
||||
{"name": "ck_maf_seat_runs_execution_stage", "sqltext": f"execution_stage IN ({STAGE3_STAGES})"},
|
||||
],
|
||||
})
|
||||
new = FakeInspector({
|
||||
"maf_roundtable_seat_runs": [
|
||||
{"name": "ck_maf_seat_runs_execution_stage", "sqltext": f"execution_stage IN ({mod.STAGE4_STAGES})"},
|
||||
],
|
||||
})
|
||||
none = FakeInspector({})
|
||||
|
||||
assert mod._has_stage4_execution_stage_check(old) is False
|
||||
assert mod._has_stage4_execution_stage_check(new) is True
|
||||
assert mod._has_stage4_execution_stage_check(none) is False
|
||||
assert mod._execution_stage_check_sql(none) is None
|
||||
@@ -250,6 +250,9 @@ async def test_workflow_projection_whitelists_user_visible_event_data(store):
|
||||
"payload": {
|
||||
"content": "answer",
|
||||
"metadata": {"thread_id": "thread-secret", "deerflow_run_id": "df-secret"},
|
||||
# A genuine seat delivery stage — only these may project a
|
||||
# seat.delivery.submitted companion event.
|
||||
"execution_stage": "expert",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -291,6 +291,9 @@ async def v2_svc(tmp_path):
|
||||
task_payload_json={
|
||||
"maf_v2": {
|
||||
"dispatch_contract_enabled": True,
|
||||
# Stage-3 topology tests freeze the pre-Stage-4 behavior:
|
||||
# plain seat fan-out, no quality gate.
|
||||
"quality_gate_enabled": False,
|
||||
"candidate_set": {
|
||||
"manager_agent_id": MANAGER,
|
||||
"candidate_seat_agent_ids": ["seat-a", "seat-b", "seat-c"],
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Stage 4 (R3-Q) P1: real conversation SSE projection tests.
|
||||
|
||||
The Stage-4 quality events must reach the USER-VISIBLE conversation event
|
||||
stream (``maf_roundtable_conversation_events``) — not just the per-run audit
|
||||
table — and they must reach it safely:
|
||||
|
||||
1. all five Stage-4 event types are projected into the conversation table;
|
||||
2. each source audit event is projected exactly ONCE (idempotent re-bridge);
|
||||
3. only whitelisted fields are exposed in ``data``;
|
||||
4. thread ids, run ids, internal action ids and tool parameters never leak;
|
||||
5. ordinary manager / finalizer / review messages are never mislabelled as
|
||||
``seat.delivery.submitted`` (only genuine seat delivery stages are).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.conversation_service import MafConversationService
|
||||
from app.maf_roundtable.event_normalizer import (
|
||||
make_manager_review_decision,
|
||||
make_seat_delivery_submitted,
|
||||
make_seat_reassigned,
|
||||
make_seat_rework_assigned,
|
||||
make_stage_quality_gate,
|
||||
make_turn_completed,
|
||||
)
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
MafRoundtableConversationRow,
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
|
||||
class FakeMafConfig:
|
||||
enabled = True
|
||||
default_mode = "structured"
|
||||
default_model_name = "test-model"
|
||||
default_reasoning_effort = None
|
||||
default_manager_agent_id = "mgr-agent"
|
||||
default_max_rounds = 8
|
||||
default_write_policy = "confirm_high_risk"
|
||||
admin_diagnostic_access = True
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def proj_env(tmp_path):
|
||||
"""Store + conversation service + one conversation bound to one run."""
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'proj.db'}")
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
import deerflow.persistence.maf_roundtable.model # noqa: F401
|
||||
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
svc = MafConversationService(store, maf_config=FakeMafConfig(), event_hub=None)
|
||||
|
||||
conv = await store.conversations.create_conversation({
|
||||
"owner_user_id": "u1", "title": "projection", "task_context": None,
|
||||
})
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="proj-sess", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="running", write_policy="auto",
|
||||
manager_agent_id="manager-1", max_rounds=8,
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="proj-run", session_id="proj-sess", status="running",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
row = await session.get(MafRoundtableConversationRow, conv["id"])
|
||||
row.internal_session_id = "proj-sess"
|
||||
await session.commit()
|
||||
|
||||
yield svc, store, conv["id"]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _stage4_events() -> list[tuple[str, dict]]:
|
||||
"""(source_audit_event_id, event) for the five Stage-4 quality events.
|
||||
|
||||
Payloads are deliberately seeded with sensitive fields that must be
|
||||
stripped by the projection whitelist.
|
||||
"""
|
||||
sensitive = {
|
||||
"thread_id": "thread-secret",
|
||||
"run_id": "run-secret",
|
||||
"session_id": "sess-secret",
|
||||
"action_id": "action-secret",
|
||||
"approval_id": "approval-secret",
|
||||
"target_key": "hmac-target-key",
|
||||
"idempotency_key": "idem-secret",
|
||||
"tool_arguments": {"url": "http://internal", "token": "abc"},
|
||||
}
|
||||
|
||||
def _with_sensitive(event: dict) -> dict:
|
||||
event = dict(event)
|
||||
event["payload"] = {**event.get("payload", {}), **sensitive}
|
||||
return event
|
||||
|
||||
return [
|
||||
("audit-delivery", _with_sensitive(make_seat_delivery_submitted(
|
||||
"seat-a", 1, task_key="01:seat-a", attempt=1,
|
||||
deterministic_passed=False, failure_codes=["below_min_length"],
|
||||
))),
|
||||
("audit-review", _with_sensitive(make_manager_review_decision(
|
||||
"seat-a", 1, task_key="01:seat-a", decision="rework",
|
||||
reason="结论过短", source="manager", attempt=1,
|
||||
))),
|
||||
("audit-rework", _with_sensitive(make_seat_rework_assigned(
|
||||
"seat-a", 1, task_key="01:seat-a", rework_count=1,
|
||||
requirements=["请补充证据"],
|
||||
))),
|
||||
("audit-reassigned", _with_sensitive(make_seat_reassigned(
|
||||
"seat-c", 1, task_key="01:seat-a", from_agent_id="seat-a",
|
||||
reassign_count=1,
|
||||
))),
|
||||
("audit-gate", _with_sensitive(make_stage_quality_gate(
|
||||
"manager-1", 1, decision="passed_with_gaps", total=2,
|
||||
accepted=1, controlled_gap=1, blocked_required=[],
|
||||
))),
|
||||
]
|
||||
|
||||
|
||||
_FORBIDDEN_KEYS = {
|
||||
"thread_id", "run_id", "session_id", "action_id", "approval_id",
|
||||
"target_key", "idempotency_key", "tool_arguments", "params", "arguments",
|
||||
"capability_snapshot", "secret", "token",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage4_events_projected_into_conversation_stream(proj_env):
|
||||
svc, store, conv_id = proj_env
|
||||
|
||||
for source_id, event in _stage4_events():
|
||||
written = await svc.bridge_workflow_event(
|
||||
run_id="proj-run", event=event, source_audit_event_id=source_id,
|
||||
)
|
||||
assert written is True, f"{event['event_type']} was not projected"
|
||||
|
||||
events = await store.conversation_events.list_events(conv_id, after_seq=0)
|
||||
types = [e["type"] for e in events]
|
||||
assert "seat.delivery.submitted" in types
|
||||
assert "manager.review.decision" in types
|
||||
assert "seat.rework.assigned" in types
|
||||
assert "seat.reassigned" in types
|
||||
assert "stage.quality_gate" in types
|
||||
|
||||
# Whitelisted fields survive; the projection carries the quality payload.
|
||||
delivery = next(e for e in events if e["type"] == "seat.delivery.submitted")
|
||||
assert delivery["data"].get("task_key") == "01:seat-a"
|
||||
assert delivery["data"].get("deterministic_passed") is False
|
||||
assert delivery["data"].get("failure_codes") == ["below_min_length"]
|
||||
review = next(e for e in events if e["type"] == "manager.review.decision")
|
||||
assert review["data"].get("decision") == "rework"
|
||||
assert review["data"].get("source") == "manager"
|
||||
gate = next(e for e in events if e["type"] == "stage.quality_gate")
|
||||
assert gate["data"].get("decision") == "passed_with_gaps"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage4_projection_is_idempotent_per_source_event(proj_env):
|
||||
svc, store, conv_id = proj_env
|
||||
|
||||
# Bridge every event, then bridge the SAME audit events again (retry /
|
||||
# reconcile after restart). Each source event projects exactly once.
|
||||
for _round in range(2):
|
||||
for source_id, event in _stage4_events():
|
||||
await svc.bridge_workflow_event(
|
||||
run_id="proj-run", event=event, source_audit_event_id=source_id,
|
||||
)
|
||||
|
||||
events = await store.conversation_events.list_events(conv_id, after_seq=0)
|
||||
projected = [
|
||||
e for e in events
|
||||
if e["type"] in {
|
||||
"seat.delivery.submitted", "manager.review.decision",
|
||||
"seat.rework.assigned", "seat.reassigned", "stage.quality_gate",
|
||||
}
|
||||
]
|
||||
assert len(projected) == 5, f"expected exactly 5 projections, got {len(projected)}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage4_projection_exposes_only_whitelist_fields(proj_env):
|
||||
svc, store, conv_id = proj_env
|
||||
|
||||
for source_id, event in _stage4_events():
|
||||
await svc.bridge_workflow_event(
|
||||
run_id="proj-run", event=event, source_audit_event_id=source_id,
|
||||
)
|
||||
|
||||
events = await store.conversation_events.list_events(conv_id, after_seq=0)
|
||||
assert events
|
||||
for e in events:
|
||||
data = e.get("data") or {}
|
||||
leaked = _FORBIDDEN_KEYS & set(data.keys())
|
||||
assert not leaked, f"{e['type']} leaks forbidden keys: {leaked}"
|
||||
# Defense in depth: forbidden VALUES never appear anywhere either.
|
||||
blob = str(data)
|
||||
assert "thread-secret" not in blob
|
||||
assert "run-secret" not in blob
|
||||
assert "hmac-target-key" not in blob
|
||||
assert "idem-secret" not in blob
|
||||
assert "action-secret" not in blob
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_and_finalizer_messages_not_seat_deliveries(proj_env):
|
||||
svc, store, conv_id = proj_env
|
||||
|
||||
# Ordinary manager-side messages: none of them may become a seat delivery.
|
||||
non_delivery_stages = [
|
||||
"manager_dispatch", "manager_review", "cross_review",
|
||||
"cross_review_repair", "finalizer", "format_repair", "planner",
|
||||
]
|
||||
for i, stage in enumerate(non_delivery_stages):
|
||||
await svc.bridge_workflow_event(
|
||||
run_id="proj-run",
|
||||
event=make_turn_completed(
|
||||
"manager-1", i + 1, content=f"{stage} message",
|
||||
execution_stage=stage,
|
||||
),
|
||||
source_audit_event_id=f"audit-msg-{stage}",
|
||||
)
|
||||
# A completed message WITHOUT a stage (legacy audit row) is fail-closed:
|
||||
# it must not be guessed to be a seat delivery either.
|
||||
await svc.bridge_workflow_event(
|
||||
run_id="proj-run",
|
||||
event=make_turn_completed("manager-1", 99, content="unknown stage"),
|
||||
source_audit_event_id="audit-msg-nostage",
|
||||
)
|
||||
|
||||
# A genuine seat delivery stage still projects both events.
|
||||
await svc.bridge_workflow_event(
|
||||
run_id="proj-run",
|
||||
event=make_turn_completed(
|
||||
"seat-a", 1, content='{"summary": "交付"}', execution_stage="expert",
|
||||
),
|
||||
source_audit_event_id="audit-msg-expert",
|
||||
)
|
||||
|
||||
events = await store.conversation_events.list_events(conv_id, after_seq=0)
|
||||
deliveries = [e for e in events if e["type"] == "seat.delivery.submitted"]
|
||||
assert len(deliveries) == 1, (
|
||||
"only the expert seat turn may project seat.delivery.submitted, got "
|
||||
f"{len(deliveries)}"
|
||||
)
|
||||
assert deliveries[0]["data"].get("agent_id") == "seat-a"
|
||||
|
||||
# All ordinary messages still appear as agent messages.
|
||||
completions = [e for e in events if e["type"] == "agent.message.complete"]
|
||||
assert len(completions) == len(non_delivery_stages) + 2 # + nostage + expert
|
||||
@@ -0,0 +1,748 @@
|
||||
"""Stage 4 (R3-Q) runner-level tests: seat task quality loop + phase gate.
|
||||
|
||||
Drives the REAL workflow (QualitySeatTasksExecutor → QualityGateExecutor →
|
||||
CrossReviewExecutor → ApprovalGateExecutor → FinalizerExecutor) through
|
||||
WorkflowRunner.execute() with a fake adapter, proving the acceptance
|
||||
criteria end-to-end against the real database:
|
||||
|
||||
- a good delivery is accepted and the run completes with quality_decision=accept;
|
||||
- a short / evidence-free delivery is sent back, reworked up to the frozen cap,
|
||||
reassigned to a backup seat once, and finally ends as a traceable
|
||||
controlled_gap that is carried into the final report (never as a fact);
|
||||
- an ``ask_user`` fallback pauses the run in waiting_input;
|
||||
- a required task left non-terminal blocks the phase quality gate;
|
||||
- a missing/invalid manager review NEVER auto-accepts: the server forces
|
||||
system reworks and exhausts into a controlled gap (source=system events).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from maf_test_helpers import make_maf_service
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.deerflow_agent_adapter import SeatResponse
|
||||
from app.maf_roundtable.workflow_executors import QualityGateExecutor, RoundtableState
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
Base,
|
||||
MafRoundtableParticipantRow,
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
MANAGER = "manager-1"
|
||||
SEATS = ["seat-a", "seat-b", "seat-c"]
|
||||
|
||||
# ``@TOOL_REF@`` is replaced per-turn with ``tool:kb_search:<turn>`` — the
|
||||
# trusted evidence ref created by the seat's (fake) successful tool call, so a
|
||||
# good delivery cites ONLY evidence the fact ledger actually backs (P1).
|
||||
GOOD_DELIVERY = json.dumps({
|
||||
"summary": "结论:该市场存在明确增长机会,建议优先进入华东区域并配套本地化运营资源投入,同时预留华南试点预算",
|
||||
"findings": ["依据一:华东区销售数据连续三个月增长", "依据二:竞品在华东投放低于均值"],
|
||||
"evidence_refs": ["@TOOL_REF@"],
|
||||
"open_questions": [],
|
||||
"self_check": "已覆盖全部验收标准",
|
||||
})
|
||||
SHORT_DELIVERY = json.dumps({"summary": "好", "findings": []})
|
||||
|
||||
DISPATCH_PARALLEL = {
|
||||
"intent_type": "parallel_consultation",
|
||||
"selected_seat_agent_ids": ["seat-a", "seat-b"],
|
||||
"assignments": [
|
||||
{"agent_id": "seat-a", "task_goal": "市场增长机会分析", "delivery_requirements": ["证据", "优先级"], "order": 1},
|
||||
{"agent_id": "seat-b", "task_goal": "竞品定价分析", "delivery_requirements": ["证据", "建议"], "order": 2},
|
||||
],
|
||||
"execution_mode": "parallel",
|
||||
"convergence_path": ["manager synthesis"],
|
||||
"reason": "需要两位专家",
|
||||
}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def q_svc(tmp_path):
|
||||
"""Service with a V2 session: manager + 3 candidates, quality gate ON."""
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'q4.db'}", echo=False)
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
service = make_maf_service(store)
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="q-sess", user_id="u1", title="q", task_text="分析市场趋势",
|
||||
mode="structured", status="pending", write_policy="auto",
|
||||
manager_agent_id=MANAGER, max_rounds=8, created_at=now, updated_at=now,
|
||||
task_payload_json={
|
||||
"maf_v2": {
|
||||
"dispatch_contract_enabled": True,
|
||||
"quality_gate_enabled": True,
|
||||
"candidate_set": {
|
||||
"manager_agent_id": MANAGER,
|
||||
"candidate_seat_agent_ids": list(SEATS),
|
||||
"mode": "structured",
|
||||
"max_rounds": 8,
|
||||
},
|
||||
},
|
||||
},
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="q-run", session_id="q-sess", status="pending",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
for i, agent_id in enumerate([MANAGER, *SEATS]):
|
||||
session.add(MafRoundtableParticipantRow(
|
||||
id=f"qp-{i}", session_id="q-sess", agent_id=agent_id,
|
||||
role="manager" if agent_id == MANAGER else "seat",
|
||||
sort_order=i, created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
yield service
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _make_q_runner(
|
||||
q_svc,
|
||||
monkeypatch,
|
||||
*,
|
||||
deliveries: dict[str, list[str]],
|
||||
review_plan: list[dict[str, dict[str, object]]],
|
||||
fallback_policy: str = "reassign_then_gap",
|
||||
cross_review_ok: bool = True,
|
||||
emit_real: bool = False,
|
||||
config_overrides: dict | None = None,
|
||||
):
|
||||
"""Runner with a fake adapter driven by explicit delivery/review plans.
|
||||
|
||||
``deliveries``: agent_id → list of JSON payloads, one per delivery attempt.
|
||||
``review_plan``: one entry per quality round → {agent_id: {"decision", ...}}.
|
||||
"""
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
app = SimpleNamespace(state=SimpleNamespace(maf_roundtable_service=q_svc, checkpointer=None))
|
||||
runner = WorkflowRunner(app, service=q_svc, user_id="u1")
|
||||
|
||||
calls: list[tuple[str, str, int]] = []
|
||||
stage_messages: dict[str, list[str]] = {}
|
||||
delivery_idx: dict[str, int] = {}
|
||||
review_round: list[int] = [0]
|
||||
|
||||
cfg_values = {
|
||||
"seat_timeout_seconds": 5,
|
||||
"schema_repair_enabled": True,
|
||||
"quality_gate_enabled": True,
|
||||
"max_rework_attempts": 2,
|
||||
"max_reassignments": 1,
|
||||
"min_delivery_chars": 40,
|
||||
"min_delivery_evidence": 1,
|
||||
"default_fallback_policy": fallback_policy,
|
||||
}
|
||||
cfg_values.update(config_overrides or {})
|
||||
fake_cfg = SimpleNamespace(maf_roundtable=SimpleNamespace(**cfg_values))
|
||||
monkeypatch.setattr("deerflow.config.get_app_config", lambda: fake_cfg)
|
||||
|
||||
async def _run_seat(**kwargs):
|
||||
stage = kwargs.get("execution_stage", "")
|
||||
agent_id = kwargs["agent_id"]
|
||||
turn_index = kwargs["turn_index"]
|
||||
calls.append((stage, agent_id, turn_index))
|
||||
stage_messages.setdefault(stage, []).append(str(kwargs.get("message") or ""))
|
||||
if stage == "manager_dispatch":
|
||||
return SeatResponse(agent_id=agent_id, turn_index=0, content=json.dumps(DISPATCH_PARALLEL), tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1})
|
||||
if stage in ("expert", "rework"):
|
||||
seq = deliveries.get(agent_id) or [GOOD_DELIVERY]
|
||||
idx = delivery_idx.get(agent_id, 0)
|
||||
delivery_idx[agent_id] = idx + 1
|
||||
content = seq[min(idx, len(seq) - 1)]
|
||||
# The seat ran a real (fake) tool this turn: its successful result
|
||||
# becomes the trusted evidence ref the delivery may cite.
|
||||
content = content.replace("@TOOL_REF@", f"tool:kb_search:{turn_index}")
|
||||
return SeatResponse(
|
||||
agent_id=agent_id,
|
||||
turn_index=turn_index,
|
||||
content=content,
|
||||
tool_calls=[{"id": f"tc-{agent_id}-{turn_index}", "name": "kb_search"}],
|
||||
usage={"input_tokens": 1, "output_tokens": 1},
|
||||
)
|
||||
if stage == "manager_review":
|
||||
rnd = review_round[0]
|
||||
review_round[0] = rnd + 1
|
||||
plan = review_plan[rnd] if rnd < len(review_plan) else {}
|
||||
decisions = [
|
||||
{
|
||||
"agent_id": agent,
|
||||
"decision": d.get("decision", "accept"),
|
||||
"reason": str(d.get("reason") or "r")[:1000],
|
||||
"revision_requirements": list(d.get("requirements") or []),
|
||||
}
|
||||
for agent, d in plan.items()
|
||||
]
|
||||
return SeatResponse(agent_id=agent_id, turn_index=turn_index, content=json.dumps({"decisions": decisions}), tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1})
|
||||
if stage == "cross_review":
|
||||
content = (
|
||||
json.dumps({"summary": "ok", "evidence_ok": True, "confirmed_candidates": []})
|
||||
if cross_review_ok
|
||||
else json.dumps({"summary": "有缺口", "evidence_ok": False, "gaps": ["证据不足"], "confirmed_candidates": []})
|
||||
)
|
||||
return SeatResponse(agent_id=agent_id, turn_index=turn_index, content=content, tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1})
|
||||
if stage == "finalizer":
|
||||
return SeatResponse(agent_id=agent_id, turn_index=turn_index, content="final report", tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1})
|
||||
return SeatResponse(agent_id=agent_id, turn_index=turn_index, content=GOOD_DELIVERY, tool_calls=[], usage={"input_tokens": 1, "output_tokens": 1})
|
||||
|
||||
ensure_threads_calls: list[list[str]] = []
|
||||
|
||||
async def _ensure_threads(session_id, run_id, participants):
|
||||
agent_ids = [p["agent_id"] for p in participants]
|
||||
ensure_threads_calls.append(agent_ids)
|
||||
result = {}
|
||||
for participant in participants:
|
||||
agent_id = participant["agent_id"]
|
||||
thread_id = f"thread-{agent_id}"
|
||||
await q_svc.store.participants.bind_thread(session_id, agent_id, thread_id=thread_id)
|
||||
result[agent_id] = thread_id
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(runner._adapter, "run_seat_turn", AsyncMock(side_effect=_run_seat))
|
||||
monkeypatch.setattr(runner._adapter, "ensure_threads", AsyncMock(side_effect=_ensure_threads))
|
||||
if emit_real:
|
||||
real_emit_critical = runner._adapter._emit_critical
|
||||
monkeypatch.setattr(runner._adapter, "_emit_critical", AsyncMock(side_effect=real_emit_critical))
|
||||
else:
|
||||
monkeypatch.setattr(runner._adapter, "_emit_critical", AsyncMock())
|
||||
monkeypatch.setattr(runner._adapter, "_emit", AsyncMock())
|
||||
runner.stage_messages = stage_messages
|
||||
return runner, calls, ensure_threads_calls
|
||||
|
||||
|
||||
async def _run(q_svc, monkeypatch, **kwargs):
|
||||
runner, calls, ensure_threads_calls = _make_q_runner(q_svc, monkeypatch, **kwargs)
|
||||
session = await q_svc.store.sessions.get_session("q-sess", user_id="u1")
|
||||
run = await q_svc.store.runs.get_run("q-run")
|
||||
await runner.execute(session, run)
|
||||
final_run = await q_svc.store.runs.get_run("q-run")
|
||||
final_session = await q_svc.store.sessions.get_session("q-sess", user_id="u1")
|
||||
return runner, calls, ensure_threads_calls, final_run, final_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_accepts_good_deliveries(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
_, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [GOOD_DELIVERY], "seat-b": [GOOD_DELIVERY]},
|
||||
review_plan=[{"seat-a": {"decision": "accept"}, "seat-b": {"decision": "accept"}}],
|
||||
)
|
||||
|
||||
assert final_run["status"] == "completed"
|
||||
assert final_session["status"] == "completed"
|
||||
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
assert set(by_key) == {"01:seat-a", "02:seat-b"}
|
||||
assert all(t["status"] == "accepted" for t in tasks)
|
||||
assert all(t["attempt_count"] == 1 for t in tasks)
|
||||
assert all(t["rework_count"] == 0 for t in tasks)
|
||||
|
||||
checkpoint = final_run.get("maf_checkpoint") or {}
|
||||
assert checkpoint.get("seat_task_statuses", {}).get("01:seat-a", {}).get("status") == "accepted"
|
||||
assert checkpoint.get("controlled_gaps") == []
|
||||
|
||||
result = final_session.get("final_result") or {}
|
||||
assert result.get("quality_decision") == "accept"
|
||||
assert result.get("controlled_gaps") == []
|
||||
|
||||
review_stages = [s for s, a, t in calls if s == "manager_review"]
|
||||
assert len(review_stages) == 1, "one batched manager review round"
|
||||
|
||||
|
||||
# ── 2. short answer → reworks → reassign → controlled gap ───────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_rework_reassign_controlled_gap(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
# seat-a stays short across every attempt; seat-b is always good.
|
||||
# Manager keeps returning return_for_rework for seat-a (cap 2), then seat-a
|
||||
# is reassigned to seat-c whose delivery also fails → controlled gap.
|
||||
_, calls, ensure_threads_calls, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={
|
||||
"seat-a": [SHORT_DELIVERY, SHORT_DELIVERY, SHORT_DELIVERY, SHORT_DELIVERY],
|
||||
"seat-b": [GOOD_DELIVERY],
|
||||
"seat-c": [SHORT_DELIVERY],
|
||||
},
|
||||
review_plan=[
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "太短", "requirements": ["请补充证据"]}, "seat-b": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "仍太短", "requirements": ["请展开分析"]}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "第三次不合格"}},
|
||||
{"seat-c": {"decision": "return_for_rework", "reason": "新席位也短"}},
|
||||
],
|
||||
)
|
||||
|
||||
# Run completes via the gap path (controlled gaps do NOT block the phase).
|
||||
assert final_run["status"] == "completed"
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
|
||||
task_a = by_key["01:seat-a"]
|
||||
assert task_a["status"] == "controlled_gap"
|
||||
assert task_a["attempt_count"] == 4 # seat-a 3 + seat-c 1
|
||||
assert task_a["rework_count"] == 2 # capped at max_rework_attempts
|
||||
assert task_a["reassign_count"] == 1 # exactly one reassignment
|
||||
assert task_a["agent_id"] == "seat-c" # reassigned to backup
|
||||
assert task_a["original_agent_id"] == "seat-a"
|
||||
assert task_a["gap"] is not None
|
||||
assert "不得作为已验证事实" in task_a["gap"]["risk"]
|
||||
|
||||
task_b = by_key["02:seat-b"]
|
||||
assert task_b["status"] == "accepted"
|
||||
assert task_b["attempt_count"] == 1
|
||||
|
||||
# seat-c received a thread (backup handover) and delivered exactly once.
|
||||
assert "seat-c" in [aid for call in ensure_threads_calls for aid in call]
|
||||
seat_c_turns = [(s, a, t) for s, a, t in calls if a == "seat-c" and s in ("expert", "rework")]
|
||||
assert len(seat_c_turns) == 1
|
||||
|
||||
# The quality gate passed with gaps and the final report carries them.
|
||||
checkpoint = final_run.get("maf_checkpoint") or {}
|
||||
gaps = checkpoint.get("controlled_gaps") or []
|
||||
assert len(gaps) == 1
|
||||
assert gaps[0]["task_key"] == "01:seat-a"
|
||||
|
||||
result = final_session.get("final_result") or {}
|
||||
assert result.get("quality_decision") == "controlled_gap"
|
||||
assert len(result.get("controlled_gaps") or []) == 1
|
||||
assert result.get("degraded") is True
|
||||
|
||||
# Manager review was batched once per quality round.
|
||||
review_stages = [t for s, a, t in calls if s == "manager_review"]
|
||||
assert len(review_stages) == 4
|
||||
|
||||
|
||||
# ── 3. ask_user fallback pauses in waiting_input ────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_ask_user_waits_for_input(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
_, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [SHORT_DELIVERY, SHORT_DELIVERY, SHORT_DELIVERY], "seat-b": [GOOD_DELIVERY]},
|
||||
review_plan=[
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "太短"}, "seat-b": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "仍太短"}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "第三次不合格"}},
|
||||
],
|
||||
fallback_policy="ask_user",
|
||||
)
|
||||
|
||||
assert final_run["status"] == "waiting_input"
|
||||
assert final_session["status"] == "waiting_input"
|
||||
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
assert by_key["01:seat-a"]["status"] == "exhausted"
|
||||
assert by_key["02:seat-b"]["status"] == "accepted"
|
||||
|
||||
# The finalizer must never run on the ask_user path.
|
||||
finalizer_stages = [s for s, a, t in calls if s == "finalizer"]
|
||||
assert finalizer_stages == []
|
||||
# No finished_at for a non-terminal pause.
|
||||
assert final_run.get("finished_at") is None
|
||||
|
||||
|
||||
# ── 4. phase gate blocks on non-terminal required task ──────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quality_gate_blocks_non_terminal_required_task(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
state = RoundtableState(
|
||||
session_id="q-sess", run_id="q-run", task_text="t",
|
||||
mode="structured", model_name=None, reasoning_effort=None,
|
||||
max_rounds=8, write_policy="auto", manager_agent_id=MANAGER,
|
||||
seat_agent_ids=["seat-a"], thread_map={}, capability_snapshot=None,
|
||||
seat_timeout_seconds=5, schema_repair_enabled=True,
|
||||
intent_type="parallel_consultation",
|
||||
dispatch_plan={"execution_mode": "parallel"},
|
||||
dispatch_contract_enabled=True, quality_gate_enabled=True,
|
||||
)
|
||||
state.seat_task_statuses = {
|
||||
"01:seat-a": {
|
||||
"task_key": "01:seat-a", "status": "submitted",
|
||||
"agent_id": "seat-a", "required": True,
|
||||
},
|
||||
}
|
||||
adapter = SimpleNamespace(
|
||||
_service=SimpleNamespace(store=SimpleNamespace(seat_tasks=SimpleNamespace())),
|
||||
_emit_critical=AsyncMock(),
|
||||
)
|
||||
gate = QualityGateExecutor(adapter, state)
|
||||
ctx = SimpleNamespace(send_message=AsyncMock(), yield_output=AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="quality gate blocked"):
|
||||
await gate.handle({}, ctx)
|
||||
|
||||
# The blocked gate must not masquerade as passed.
|
||||
assert state.status != "completed"
|
||||
|
||||
|
||||
# ── 5. invalid/missing manager review NEVER auto-accepts ───────────────────
|
||||
|
||||
|
||||
def _good_variant(n: int) -> str:
|
||||
"""A deterministic-pass delivery whose conclusion differs per attempt, so
|
||||
duplicate detection never interferes with the missing-review behaviour."""
|
||||
return json.dumps({
|
||||
"summary": (
|
||||
f"结论{n}:该市场存在明确增长机会,建议优先进入华东区域并配套本地化运营资源投入,"
|
||||
f"同时预留华南试点预算,第{n}次补充细化执行路径与里程碑安排"
|
||||
),
|
||||
"findings": [f"依据{n}:华东区销售数据连续三个月增长", "依据二:竞品在华东投放低于均值"],
|
||||
"evidence_refs": ["@TOOL_REF@"],
|
||||
"open_questions": [],
|
||||
"self_check": "已覆盖全部验收标准",
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_missing_review_never_accepts(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
# The manager review returns garbage (empty decisions) every round. Even
|
||||
# though every delivery passes the deterministic length/format/evidence
|
||||
# checks, NO task may be accepted without a valid manager review: the
|
||||
# server forces system reworks, then exhausts into controlled gaps.
|
||||
variants_a = [_good_variant(i) for i in range(1, 7)]
|
||||
variants_b = [_good_variant(i + 10) for i in range(1, 7)]
|
||||
_, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={
|
||||
"seat-a": variants_a + [_good_variant(90)],
|
||||
"seat-b": variants_b + [_good_variant(91)],
|
||||
"seat-c": [_good_variant(80)],
|
||||
},
|
||||
review_plan=[{}],
|
||||
emit_real=True,
|
||||
)
|
||||
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
assert tasks, "seat tasks must exist"
|
||||
for task in tasks:
|
||||
assert task["status"] != "accepted", (
|
||||
f"task {task['task_key']} was accepted without a valid manager review"
|
||||
)
|
||||
assert task["status"] == "controlled_gap"
|
||||
assert task["rework_count"] == 2
|
||||
assert task["reassign_count"] == 1
|
||||
decision = task.get("review_decision") or {}
|
||||
assert decision.get("source") == "system"
|
||||
assert decision.get("reason") == "manager_review_missing_or_invalid"
|
||||
|
||||
# The run still terminates (gaps do not block the phase) — but only as a
|
||||
# degraded controlled-gap result, never as a clean accept.
|
||||
assert final_run["status"] == "completed"
|
||||
result = final_session.get("final_result") or {}
|
||||
assert result.get("quality_decision") == "controlled_gap"
|
||||
assert len(result.get("controlled_gaps") or []) == len(tasks)
|
||||
|
||||
# The persisted review events carry source=system + the safe reason.
|
||||
events = await q_svc.store.events.list_events(run_id="q-run")
|
||||
review_events = [e for e in events if e["event_type"] == "manager.review.decision"]
|
||||
assert review_events
|
||||
system_events = [e for e in review_events if e["payload"].get("source") == "system"]
|
||||
assert system_events
|
||||
assert all(
|
||||
e["payload"].get("reason") == "manager_review_missing_or_invalid"
|
||||
for e in system_events
|
||||
)
|
||||
|
||||
|
||||
# ── 6. unaccepted deliveries never reach facts / review / final report ─────
|
||||
|
||||
|
||||
PLAUSIBLE_GAP_DELIVERY = json.dumps({
|
||||
# Fails below_min_length on purpose, but carries "plausible" findings and
|
||||
# an invented evidence ref — none of it may survive into the ledger, the
|
||||
# cross-review prompt or the final synthesis.
|
||||
"summary": "好",
|
||||
"findings": ["PLAUSIBLE_FINDING: 已核实华东区增长数据并确认竞品全面撤离"],
|
||||
"evidence_refs": ["plausible-ref-001"],
|
||||
"open_questions": [],
|
||||
"self_check": "ok",
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_gap_content_isolated_from_facts_and_report(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
runner, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={
|
||||
"seat-a": [PLAUSIBLE_GAP_DELIVERY],
|
||||
"seat-b": [GOOD_DELIVERY],
|
||||
"seat-c": [PLAUSIBLE_GAP_DELIVERY],
|
||||
},
|
||||
review_plan=[
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "太短"}, "seat-b": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "仍太短"}},
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "第三次不合格"}},
|
||||
{"seat-c": {"decision": "return_for_rework", "reason": "新席位也不合格"}},
|
||||
],
|
||||
)
|
||||
|
||||
assert final_run["status"] == "completed"
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
assert by_key["01:seat-a"]["status"] == "controlled_gap"
|
||||
assert by_key["02:seat-b"]["status"] == "accepted"
|
||||
|
||||
# 1) Fact ledger: the gapped seat's claims/refs are absent; the accepted
|
||||
# seat's findings ARE present (merge happens on acceptance).
|
||||
ledger = final_session.get("fact_ledger") or {}
|
||||
candidate_texts = [
|
||||
str(c.get("text") or "") for c in ledger.get("candidate_facts") or []
|
||||
]
|
||||
confirmed_texts = [
|
||||
str(f.get("text") or "") for f in ledger.get("confirmed_facts") or []
|
||||
]
|
||||
evidence_texts = [
|
||||
str(e.get("ref") or e.get("text") or "") for e in ledger.get("evidence") or []
|
||||
]
|
||||
assert not any("PLAUSIBLE_FINDING" in t for t in candidate_texts)
|
||||
assert not any("PLAUSIBLE_FINDING" in t for t in confirmed_texts)
|
||||
assert not any("plausible-ref-001" in t for t in evidence_texts)
|
||||
assert any("华东区销售数据连续三个月增长" in t for t in candidate_texts)
|
||||
|
||||
# 2) Cross-review prompt: accepted seat only, never the gapped content.
|
||||
cross_review_msgs = runner.stage_messages.get("cross_review") or []
|
||||
assert cross_review_msgs
|
||||
assert all("PLAUSIBLE_FINDING" not in m for m in cross_review_msgs)
|
||||
assert all("plausible-ref-001" not in m for m in cross_review_msgs)
|
||||
assert any("华东区销售数据连续三个月增长" in m for m in cross_review_msgs)
|
||||
|
||||
# 3) Final synthesis: accepted seat only + traceable gap risk section.
|
||||
finalizer_msgs = runner.stage_messages.get("finalizer") or []
|
||||
assert finalizer_msgs
|
||||
assert all("PLAUSIBLE_FINDING" not in m for m in finalizer_msgs)
|
||||
assert all("plausible-ref-001" not in m for m in finalizer_msgs)
|
||||
assert any("01:seat-a" in m for m in finalizer_msgs), "gap must be traceable"
|
||||
assert any("受控缺口" in m for m in finalizer_msgs)
|
||||
|
||||
# 4) Checkpoint: exactly one accepted result (the accepted seat).
|
||||
checkpoint = final_run.get("maf_checkpoint") or {}
|
||||
accepted_results = checkpoint.get("accepted_task_results") or []
|
||||
assert len(accepted_results) == 1
|
||||
assert accepted_results[0]["task_key"] == "02:seat-b"
|
||||
assert accepted_results[0]["accepted"] is True
|
||||
assert "PLAUSIBLE_FINDING" not in str(accepted_results)
|
||||
|
||||
# 5) Final result carries the gap as risk, never as a verified fact.
|
||||
result = final_session.get("final_result") or {}
|
||||
assert result.get("quality_decision") == "controlled_gap"
|
||||
gaps = result.get("controlled_gaps") or []
|
||||
assert len(gaps) == 1
|
||||
assert gaps[0]["task_key"] == "01:seat-a"
|
||||
assert "不得作为已验证事实" in gaps[0]["risk"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_events_persisted_and_projected(q_svc, monkeypatch):
|
||||
"""The Stage-4 audit events persist to maf_roundtable_events."""
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
_, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [SHORT_DELIVERY], "seat-b": [GOOD_DELIVERY]},
|
||||
review_plan=[{"seat-a": {"decision": "return_for_rework", "requirements": ["补证据"]}, "seat-b": {"decision": "accept"}}],
|
||||
emit_real=True,
|
||||
)
|
||||
|
||||
events = await q_svc.store.events.list_events(run_id="q-run")
|
||||
types = {e["event_type"] for e in events}
|
||||
assert "seat.delivery.submitted" in types
|
||||
assert "manager.review.decision" in types
|
||||
assert "seat.rework.assigned" in types
|
||||
assert "stage.quality_gate" in types
|
||||
|
||||
# The short delivery was flagged deterministically in the persisted event.
|
||||
deliveries = [e for e in events if e["event_type"] == "seat.delivery.submitted"]
|
||||
seat_a = [e for e in deliveries if e.get("agent_id") == "seat-a"]
|
||||
assert seat_a and seat_a[0]["payload"].get("deterministic_passed") is False
|
||||
|
||||
|
||||
# ── 7. P1: zero config values are honoured, not coerced to defaults ────────
|
||||
|
||||
|
||||
def test_quality_settings_honour_zero_config_values(monkeypatch):
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
fake_cfg = SimpleNamespace(maf_roundtable=SimpleNamespace(
|
||||
quality_gate_enabled=True,
|
||||
max_rework_attempts=0,
|
||||
max_reassignments=0,
|
||||
min_delivery_chars=0,
|
||||
min_delivery_evidence=0,
|
||||
default_fallback_policy="controlled_gap",
|
||||
))
|
||||
monkeypatch.setattr("deerflow.config.get_app_config", lambda: fake_cfg)
|
||||
|
||||
runner = WorkflowRunner.__new__(WorkflowRunner) # no __init__ side effects
|
||||
settings = runner._quality_settings()
|
||||
assert settings["max_rework_attempts"] == 0
|
||||
assert settings["max_reassignments"] == 0
|
||||
assert settings["min_delivery_chars"] == 0
|
||||
assert settings["min_delivery_evidence"] == 0
|
||||
assert settings["fallback_policy"] == "controlled_gap"
|
||||
|
||||
|
||||
def test_quality_settings_none_falls_back_to_defaults(monkeypatch):
|
||||
from app.maf_roundtable.workflow_runner import WorkflowRunner
|
||||
|
||||
# Attributes absent entirely (getattr → None) must fall back to defaults.
|
||||
fake_cfg = SimpleNamespace(maf_roundtable=SimpleNamespace(
|
||||
quality_gate_enabled=True,
|
||||
))
|
||||
monkeypatch.setattr("deerflow.config.get_app_config", lambda: fake_cfg)
|
||||
|
||||
runner = WorkflowRunner.__new__(WorkflowRunner)
|
||||
settings = runner._quality_settings()
|
||||
assert settings["max_rework_attempts"] == 2
|
||||
assert settings["max_reassignments"] == 1
|
||||
assert settings["min_delivery_chars"] == 40
|
||||
assert settings["min_delivery_evidence"] == 1
|
||||
assert settings["fallback_policy"] == "reassign_then_gap"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_zero_rework_cap_no_rework_granted(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
# max_rework_attempts=0 must be honoured: a failing delivery is NOT sent
|
||||
# back for rework — it exhausts straight into the fallback (controlled gap
|
||||
# here, no backup reassignment budget either).
|
||||
_, calls, _, final_run, final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [SHORT_DELIVERY], "seat-b": [GOOD_DELIVERY]},
|
||||
review_plan=[
|
||||
{"seat-a": {"decision": "return_for_rework", "reason": "太短"}, "seat-b": {"decision": "accept"}},
|
||||
],
|
||||
config_overrides={"max_rework_attempts": 0, "max_reassignments": 0},
|
||||
)
|
||||
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
task_a = by_key["01:seat-a"]
|
||||
assert task_a["status"] == "controlled_gap"
|
||||
assert task_a["attempt_count"] == 1, "no rework delivery may happen with a 0 cap"
|
||||
assert task_a["rework_count"] == 0
|
||||
assert task_a["reassign_count"] == 0
|
||||
# Only the initial delivery + the (ignored) rework request — no rework turn.
|
||||
expert_turns_a = [s for s, a, _t in calls if a == "seat-a" and s in ("expert", "rework")]
|
||||
assert len(expert_turns_a) == 1
|
||||
|
||||
result = final_session.get("final_result") or {}
|
||||
assert result.get("quality_decision") == "controlled_gap"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_zero_min_evidence_accepts_refless_delivery(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
# min_delivery_evidence=0 must be honoured: a conclusion-only delivery with
|
||||
# no evidence refs passes the deterministic evidence gate.
|
||||
no_ref_delivery = json.dumps({
|
||||
"summary": "结论:该市场存在明确增长机会,建议优先进入华东区域并配套本地化运营资源投入,同时预留华南试点预算",
|
||||
"findings": [],
|
||||
"evidence_refs": [],
|
||||
"open_questions": [],
|
||||
"self_check": "ok",
|
||||
})
|
||||
_, _calls, _, final_run, _final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [no_ref_delivery], "seat-b": [no_ref_delivery]},
|
||||
review_plan=[{"seat-a": {"decision": "accept"}, "seat-b": {"decision": "accept"}}],
|
||||
config_overrides={"min_delivery_evidence": 0},
|
||||
)
|
||||
|
||||
assert final_run["status"] == "completed"
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
assert all(t["status"] == "accepted" for t in tasks)
|
||||
|
||||
|
||||
# ── 8. P1: restart recovery re-submission hits duplicate via digest ────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_quality_restart_resubmission_hits_duplicate_digest(q_svc, monkeypatch):
|
||||
pytest.importorskip("agent_framework")
|
||||
|
||||
from app.maf_roundtable.quality_contracts import build_task_contract, conclusion_digest
|
||||
|
||||
good = json.loads(GOOD_DELIVERY.replace("@TOOL_REF@", "tool:kb_search:1"))
|
||||
digest = conclusion_digest(good["summary"])
|
||||
|
||||
# Simulate the pre-restart state: seat-a's first delivery was rejected and
|
||||
# its conclusion digest was persisted to the task history; then the process
|
||||
# restarted. The task row (with history) survives in the DB.
|
||||
contract = build_task_contract(
|
||||
task_goal="市场增长机会分析", delivery_requirements=["证据", "优先级"],
|
||||
max_rework_attempts=2, fallback_policy="reassign_then_gap",
|
||||
)
|
||||
seeded = await q_svc.store.seat_tasks.create_task(
|
||||
session_id="q-sess", run_id="q-run", task_key="01:seat-a",
|
||||
agent_id="seat-a", contract=contract.model_dump(), required=True,
|
||||
)
|
||||
await q_svc.store.seat_tasks.update_task(seeded["id"], {
|
||||
"status": "rework",
|
||||
"attempt_count": 1,
|
||||
"rework_count": 1,
|
||||
"history": [{
|
||||
"attempt": 1, "agent_id": "seat-a", "round": 1,
|
||||
"schema_status": "ok", "failure_codes": ["below_min_length"],
|
||||
"conclusion_sha256": digest,
|
||||
}],
|
||||
})
|
||||
|
||||
# After the restart the seat re-submits the SAME conclusion. It must be
|
||||
# flagged duplicate_content from the persisted digest — even though the
|
||||
# in-memory prior_conclusions starts empty on a fresh process.
|
||||
_, _calls, _, final_run, _final_session = await _run(
|
||||
q_svc, monkeypatch,
|
||||
deliveries={"seat-a": [GOOD_DELIVERY], "seat-b": [GOOD_DELIVERY]},
|
||||
review_plan=[
|
||||
{"seat-a": {"decision": "accept"}, "seat-b": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "accept"}},
|
||||
{"seat-a": {"decision": "accept"}},
|
||||
],
|
||||
)
|
||||
|
||||
tasks = await q_svc.store.seat_tasks.list_by_run("q-run")
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
task_a = by_key["01:seat-a"]
|
||||
# The re-submission was deterministically flagged as a duplicate.
|
||||
check = task_a.get("deterministic_check") or {}
|
||||
history_codes = [
|
||||
code
|
||||
for h in (task_a.get("history") or [])
|
||||
for code in (h.get("failure_codes") or [])
|
||||
]
|
||||
assert "duplicate_content" in (check.get("failure_codes") or []) or "duplicate_content" in history_codes
|
||||
# And a duplicate can never be accepted, even with the manager saying yes.
|
||||
assert task_a["status"] != "accepted"
|
||||
assert final_run["status"] == "completed"
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Stage 4 (R3-Q) P0-4: atomic seat-task initialization and safe recovery.
|
||||
|
||||
The old initializer created tasks one by one and short-circuited with
|
||||
``if existing: return existing`` — a crash after the first create permanently
|
||||
dropped the remaining required seats on recovery, and the quality gate could
|
||||
then pass on an incomplete task set.
|
||||
|
||||
These tests prove:
|
||||
|
||||
- ``get_or_create_tasks`` creates the FULL expected set in one transaction;
|
||||
- after a simulated partial initialization (only the first task exists),
|
||||
recovery fills the missing tasks and never touches existing rows;
|
||||
- a frozen contract / lifecycle progress is NEVER overwritten;
|
||||
- two concurrent recovery calls converge on one complete set without raising
|
||||
unique-key violations;
|
||||
- ``QualitySeatTasksExecutor._load_or_create_tasks`` derives the expected set
|
||||
from the dispatch assignments and fills gaps;
|
||||
- the quality gate blocks when an expected required task is missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.maf_roundtable.workflow_executors import (
|
||||
QualityGateExecutor,
|
||||
QualitySeatTasksExecutor,
|
||||
RoundtableState,
|
||||
)
|
||||
from deerflow.persistence.maf_roundtable import make_maf_roundtable_store
|
||||
from deerflow.persistence.maf_roundtable.model import (
|
||||
Base,
|
||||
MafRoundtableRunRow,
|
||||
MafRoundtableSessionRow,
|
||||
)
|
||||
|
||||
CONTRACT_A = {
|
||||
"objective": "原始合同目标 A",
|
||||
"expected_deliverable": "结论+证据",
|
||||
"acceptance_criteria": ["证据"],
|
||||
"required": True,
|
||||
"max_rework_attempts": 2,
|
||||
"fallback_policy": "reassign_then_gap",
|
||||
}
|
||||
CONTRACT_B = {
|
||||
"objective": "试图覆盖的合同 B",
|
||||
"expected_deliverable": "别的",
|
||||
"acceptance_criteria": [],
|
||||
"required": True,
|
||||
"max_rework_attempts": 0,
|
||||
"fallback_policy": "controlled_gap",
|
||||
}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def task_store(tmp_path):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'tasks.db'}", echo=False)
|
||||
try:
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = make_maf_roundtable_store(sf)
|
||||
now = datetime(2026, 1, 1)
|
||||
async with sf() as session:
|
||||
session.add(MafRoundtableSessionRow(
|
||||
id="t-sess", user_id="u1", title="t", task_text="t",
|
||||
mode="structured", status="pending", write_policy="auto",
|
||||
manager_agent_id="manager-1", max_rounds=8,
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
session.add(MafRoundtableRunRow(
|
||||
id="t-run", session_id="t-sess", status="pending",
|
||||
created_at=now, updated_at=now,
|
||||
))
|
||||
await session.commit()
|
||||
yield store
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _expected_specs() -> list[dict]:
|
||||
return [
|
||||
{"task_key": "01:seat-a", "agent_id": "seat-a", "contract": dict(CONTRACT_A), "required": True},
|
||||
{"task_key": "02:seat-b", "agent_id": "seat-b", "contract": dict(CONTRACT_A), "required": True},
|
||||
{"task_key": "03:seat-c", "agent_id": "seat-c", "contract": dict(CONTRACT_A), "required": True},
|
||||
]
|
||||
|
||||
|
||||
# ── repository: atomic full-set creation ────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_tasks_creates_full_set(task_store):
|
||||
tasks = await task_store.seat_tasks.get_or_create_tasks(
|
||||
session_id="t-sess", run_id="t-run", expected=_expected_specs(),
|
||||
)
|
||||
assert [t["task_key"] for t in tasks] == ["01:seat-a", "02:seat-b", "03:seat-c"]
|
||||
assert all(t["status"] == "assigned" for t in tasks)
|
||||
assert all(t["contract"]["objective"] == "原始合同目标 A" for t in tasks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_tasks_recovers_partial_initialization(task_store):
|
||||
# Simulate the old crash mode: only the FIRST task survived initialization.
|
||||
await task_store.seat_tasks.create_task(
|
||||
session_id="t-sess", run_id="t-run", task_key="01:seat-a",
|
||||
agent_id="seat-a", contract=dict(CONTRACT_A), required=True,
|
||||
)
|
||||
|
||||
tasks = await task_store.seat_tasks.get_or_create_tasks(
|
||||
session_id="t-sess", run_id="t-run", expected=_expected_specs(),
|
||||
)
|
||||
# The complete set comes back — the missing seats are filled in.
|
||||
assert [t["task_key"] for t in tasks] == ["01:seat-a", "02:seat-b", "03:seat-c"]
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
# The pre-existing row kept its identity and frozen contract.
|
||||
assert by_key["01:seat-a"]["contract"]["objective"] == "原始合同目标 A"
|
||||
assert by_key["01:seat-a"]["status"] == "assigned"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_tasks_never_overwrites_frozen_contract(task_store):
|
||||
existing = await task_store.seat_tasks.create_task(
|
||||
session_id="t-sess", run_id="t-run", task_key="01:seat-a",
|
||||
agent_id="seat-c", contract=dict(CONTRACT_A), required=True,
|
||||
)
|
||||
# Lifecycle progressed (e.g. before a restart): reassignment + attempts.
|
||||
await task_store.seat_tasks.update_task(existing["id"], {
|
||||
"status": "accepted",
|
||||
"attempt_count": 3,
|
||||
"rework_count": 2,
|
||||
"reassign_count": 1,
|
||||
})
|
||||
|
||||
specs = _expected_specs()
|
||||
specs[0]["contract"] = dict(CONTRACT_B) # recovery must NOT apply this
|
||||
tasks = await task_store.seat_tasks.get_or_create_tasks(
|
||||
session_id="t-sess", run_id="t-run", expected=specs,
|
||||
)
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
kept = by_key["01:seat-a"]
|
||||
assert kept["id"] == existing["id"]
|
||||
assert kept["contract"]["objective"] == "原始合同目标 A"
|
||||
assert kept["agent_id"] == "seat-c"
|
||||
assert kept["status"] == "accepted"
|
||||
assert kept["attempt_count"] == 3
|
||||
assert kept["rework_count"] == 2
|
||||
assert kept["reassign_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_or_create_tasks_concurrent_recovery(task_store):
|
||||
# Two recoveries race on the same empty run: both must return the complete
|
||||
# set, neither may raise a unique-key violation, and exactly one row per
|
||||
# task_key may exist afterwards.
|
||||
results = await asyncio.gather(
|
||||
task_store.seat_tasks.get_or_create_tasks(
|
||||
session_id="t-sess", run_id="t-run", expected=_expected_specs(),
|
||||
),
|
||||
task_store.seat_tasks.get_or_create_tasks(
|
||||
session_id="t-sess", run_id="t-run", expected=_expected_specs(),
|
||||
),
|
||||
)
|
||||
for tasks in results:
|
||||
assert [t["task_key"] for t in tasks] == ["01:seat-a", "02:seat-b", "03:seat-c"]
|
||||
|
||||
final = await task_store.seat_tasks.list_by_run("t-run")
|
||||
keys = [t["task_key"] for t in final]
|
||||
assert sorted(keys) == ["01:seat-a", "02:seat-b", "03:seat-c"]
|
||||
assert len(keys) == len(set(keys)), "no duplicate tasks"
|
||||
|
||||
|
||||
# ── executor: expected set from dispatch assignments ────────────────────────
|
||||
|
||||
|
||||
def _make_state(**overrides) -> RoundtableState:
|
||||
kwargs = {
|
||||
"session_id": "t-sess",
|
||||
"run_id": "t-run",
|
||||
"task_text": "分析市场",
|
||||
"mode": "structured",
|
||||
"model_name": None,
|
||||
"reasoning_effort": None,
|
||||
"max_rounds": 8,
|
||||
"write_policy": "auto",
|
||||
"manager_agent_id": "manager-1",
|
||||
"seat_agent_ids": ["seat-a", "seat-b"],
|
||||
"thread_map": {},
|
||||
"capability_snapshot": None,
|
||||
"seat_timeout_seconds": 5,
|
||||
"schema_repair_enabled": True,
|
||||
"intent_type": "parallel_consultation",
|
||||
"dispatch_plan": {
|
||||
"execution_mode": "parallel",
|
||||
"assignments": [
|
||||
{"agent_id": "seat-a", "task_goal": "目标一", "delivery_requirements": ["证据"]},
|
||||
{"agent_id": "seat-b", "task_goal": "目标二", "delivery_requirements": ["建议"]},
|
||||
],
|
||||
},
|
||||
"dispatch_contract_enabled": True,
|
||||
"quality_gate_enabled": True,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return RoundtableState(**kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_executor_load_or_create_fills_missing_tasks(task_store):
|
||||
# Partial init: only the first assignment's task exists.
|
||||
await task_store.seat_tasks.create_task(
|
||||
session_id="t-sess", run_id="t-run", task_key="01:seat-a",
|
||||
agent_id="seat-a", contract=dict(CONTRACT_A), required=True,
|
||||
)
|
||||
state = _make_state()
|
||||
adapter = SimpleNamespace(_service=SimpleNamespace(store=task_store))
|
||||
executor = QualitySeatTasksExecutor(adapter, state)
|
||||
|
||||
tasks = await executor._load_or_create_tasks()
|
||||
assert [t["task_key"] for t in tasks] == ["01:seat-a", "02:seat-b"]
|
||||
by_key = {t["task_key"]: t for t in tasks}
|
||||
# Existing contract untouched; missing task created from the assignment.
|
||||
assert by_key["01:seat-a"]["contract"]["objective"] == "原始合同目标 A"
|
||||
assert by_key["02:seat-b"]["contract"]["objective"] == "目标二"
|
||||
assert by_key["02:seat-b"]["status"] == "assigned"
|
||||
|
||||
# Idempotent second recovery.
|
||||
again = await executor._load_or_create_tasks()
|
||||
assert [t["id"] for t in again] == [t["id"] for t in tasks]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quality_gate_blocks_missing_required_task(task_store):
|
||||
state = _make_state()
|
||||
# Only ONE of the two expected required tasks is represented.
|
||||
state.seat_task_statuses = {
|
||||
"01:seat-a": {
|
||||
"task_key": "01:seat-a", "status": "accepted",
|
||||
"agent_id": "seat-a", "required": True,
|
||||
},
|
||||
}
|
||||
adapter = SimpleNamespace(
|
||||
_service=SimpleNamespace(store=SimpleNamespace(seat_tasks=SimpleNamespace())),
|
||||
_emit_critical=AsyncMock(),
|
||||
)
|
||||
gate = QualityGateExecutor(adapter, state)
|
||||
ctx = SimpleNamespace(send_message=AsyncMock(), yield_output=AsyncMock())
|
||||
with pytest.raises(RuntimeError, match="quality gate blocked"):
|
||||
await gate.handle({}, ctx)
|
||||
assert state.status != "completed"
|
||||
@@ -0,0 +1,565 @@
|
||||
"""Stage 4 (R3-Q) pure-policy tests: task contracts, deterministic quality
|
||||
checks, manager review schema, and server-owned decision finalization.
|
||||
|
||||
These tests exercise ``quality_contracts`` directly (no MAF SDK, no I/O) and
|
||||
prove the weak-model protections:
|
||||
|
||||
- a delivery that failed deterministic checks can never be accepted, no matter
|
||||
what the manager model claims;
|
||||
- rework is bounded by the frozen contract's ``max_rework_attempts``;
|
||||
- reassignment is bounded by the server-configured cap;
|
||||
- an exhausted task ends as ``controlled_gap`` / ``ask_user``, never as a
|
||||
disguised success.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.maf_roundtable.quality_contracts import (
|
||||
OUTCOME_ACCEPTED,
|
||||
OUTCOME_ASK_USER,
|
||||
OUTCOME_CONTROLLED_GAP,
|
||||
OUTCOME_REASSIGNED,
|
||||
OUTCOME_REWORK,
|
||||
InvalidTaskContractError,
|
||||
MafManagerReviewItem,
|
||||
MafManagerReviewOutput,
|
||||
MafSeatDelivery,
|
||||
MafTaskContract,
|
||||
build_manager_review_prompt,
|
||||
build_task_contract,
|
||||
check_delivery_deterministic,
|
||||
conclusion_digest,
|
||||
delivery_from_seat_output,
|
||||
finalize_review_decision,
|
||||
make_controlled_gap,
|
||||
parse_manager_review,
|
||||
repair_requirements_for_codes,
|
||||
)
|
||||
|
||||
# ── task contract freezing ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_build_task_contract_freezes_from_dispatch_assignment():
|
||||
contract = build_task_contract(
|
||||
task_goal="评估竞品定价策略",
|
||||
delivery_requirements=["风险清单", "依据", "优先级", "建议"],
|
||||
required=True,
|
||||
max_rework_attempts=2,
|
||||
fallback_policy="reassign_then_gap",
|
||||
)
|
||||
assert contract.objective == "评估竞品定价策略"
|
||||
assert contract.expected_deliverable
|
||||
assert contract.acceptance_criteria == ["风险清单", "依据", "优先级", "建议"]
|
||||
assert contract.required is True
|
||||
assert contract.max_rework_attempts == 2
|
||||
assert contract.fallback_policy == "reassign_then_gap"
|
||||
|
||||
|
||||
def test_build_task_contract_rejects_empty_objective():
|
||||
try:
|
||||
build_task_contract(task_goal=" ", delivery_requirements=[])
|
||||
except InvalidTaskContractError:
|
||||
return
|
||||
raise AssertionError("empty objective must raise InvalidTaskContractError")
|
||||
|
||||
|
||||
def test_build_task_contract_strips_blank_criteria():
|
||||
contract = build_task_contract(task_goal="t", delivery_requirements=["a", " ", "b"])
|
||||
assert contract.acceptance_criteria == ["a", "b"]
|
||||
|
||||
|
||||
# ── delivery mapping ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_delivery_from_seat_output_maps_all_fields():
|
||||
delivery = delivery_from_seat_output(
|
||||
structured_output={
|
||||
"summary": "结论:竞品定价偏高",
|
||||
"findings": ["find-1", {"text": "find-2"}],
|
||||
"evidence_refs": [{"ref": "ref-a"}, "ref-b"],
|
||||
"open_questions": ["q1"],
|
||||
"self_check": "已覆盖优先级标准",
|
||||
},
|
||||
raw_content='{"summary": "结论:竞品定价偏高"}',
|
||||
)
|
||||
assert delivery.conclusion == "结论:竞品定价偏高"
|
||||
assert delivery.evidence == ["find-1", "find-2", "ref-a", "ref-b"]
|
||||
# P1: only evidence_refs can satisfy the evidence requirement.
|
||||
assert delivery.evidence_refs == ["ref-a", "ref-b"]
|
||||
assert delivery.unresolved_items == ["q1"]
|
||||
assert delivery.self_check == "已覆盖优先级标准"
|
||||
|
||||
|
||||
def test_delivery_from_seat_output_schema_invalid_is_empty():
|
||||
delivery = delivery_from_seat_output(structured_output=None, raw_content="not json")
|
||||
assert delivery.conclusion == ""
|
||||
assert delivery.evidence == []
|
||||
assert delivery.evidence_refs == []
|
||||
|
||||
|
||||
# ── deterministic checks ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _good_delivery() -> MafSeatDelivery:
|
||||
return MafSeatDelivery(
|
||||
conclusion="结论:该市场存在明确增长机会,建议优先进入华东区域并配套本地化运营资源投入,同时预留华南试点预算",
|
||||
evidence=["ref-1", "ref-2"],
|
||||
evidence_refs=["ref-1", "ref-2"],
|
||||
unresolved_items=[],
|
||||
self_check="ok",
|
||||
)
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_passes_well_formed():
|
||||
delivery = _good_delivery()
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok",
|
||||
)
|
||||
assert result["passed"] is True
|
||||
assert result["failure_codes"] == []
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_empty_content():
|
||||
result = check_delivery_deterministic(
|
||||
delivery=_good_delivery(), raw_content=" ", schema_status="ok",
|
||||
)
|
||||
assert result["passed"] is False
|
||||
assert "empty_content" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_schema_invalid():
|
||||
result = check_delivery_deterministic(
|
||||
delivery=_good_delivery(), raw_content="not json", schema_status="schema_invalid",
|
||||
)
|
||||
assert result["passed"] is False
|
||||
assert "schema_invalid" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_missing_conclusion():
|
||||
delivery = MafSeatDelivery(conclusion="", evidence=["e"], self_check="")
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery, raw_content='{"summary":""}', schema_status="ok",
|
||||
)
|
||||
assert "missing_conclusion" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_below_min_length():
|
||||
delivery = MafSeatDelivery(conclusion="好", evidence=["e"], self_check="")
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery, raw_content='{"summary":"好"}', schema_status="ok", min_length=40,
|
||||
)
|
||||
assert "below_min_length" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_missing_evidence():
|
||||
delivery = MafSeatDelivery(conclusion="这是一个足够长的结论用于通过最小长度校验", evidence=[], self_check="")
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1,
|
||||
)
|
||||
assert "missing_evidence" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_findings_alone_never_satisfy_evidence():
|
||||
# P1: findings are candidate claims only — a delivery full of findings but
|
||||
# without evidence_refs still fails the evidence requirement.
|
||||
delivery = MafSeatDelivery(
|
||||
conclusion="这是一个足够长的结论用于通过最小长度校验,附带大量候选主张作为干扰项",
|
||||
evidence=["我查到 A", "我查到 B", "find-3"],
|
||||
evidence_refs=[],
|
||||
self_check="",
|
||||
)
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1,
|
||||
)
|
||||
assert result["passed"] is False
|
||||
assert "missing_evidence" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_invented_evidence_ref_fails_unverified():
|
||||
# P1: a cited ref that the fact ledger does not back is a hallucination.
|
||||
delivery = MafSeatDelivery(
|
||||
conclusion="这是一个足够长的结论用于通过最小长度校验,引用了不存在的证据",
|
||||
evidence=["fake-ref-001"],
|
||||
evidence_refs=["fake-ref-001"],
|
||||
self_check="",
|
||||
)
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery,
|
||||
raw_content='{"summary":"x"}',
|
||||
schema_status="ok",
|
||||
min_evidence=1,
|
||||
verified_evidence_refs={"tool:kb_search:1", "ref-real"},
|
||||
)
|
||||
assert result["passed"] is False
|
||||
assert "unverified_evidence_ref" in result["failure_codes"]
|
||||
# The invented ref also cannot count toward the evidence minimum.
|
||||
assert "missing_evidence" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_check_delivery_verified_evidence_ref_passes():
|
||||
delivery = MafSeatDelivery(
|
||||
conclusion="这是一个足够长的结论用于通过最小长度校验,引用了事实账本中真实存在的证据引用,满足最小证据数量要求",
|
||||
evidence=["tool:kb_search:1"],
|
||||
evidence_refs=["tool:kb_search:1"],
|
||||
self_check="",
|
||||
)
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery,
|
||||
raw_content='{"summary":"x"}',
|
||||
schema_status="ok",
|
||||
min_evidence=1,
|
||||
verified_evidence_refs={"tool:kb_search:1"},
|
||||
)
|
||||
assert result["passed"] is True
|
||||
assert result["failure_codes"] == []
|
||||
|
||||
|
||||
def test_check_delivery_deterministic_duplicate_content_by_digest():
|
||||
delivery = MafSeatDelivery(
|
||||
conclusion="结论:与上次完全相同的内容", evidence=["e"], evidence_refs=["e"], self_check="",
|
||||
)
|
||||
result = check_delivery_deterministic(
|
||||
delivery=delivery,
|
||||
raw_content="x",
|
||||
schema_status="ok",
|
||||
prior_conclusion_digests={conclusion_digest("结论:与上次完全相同的内容")},
|
||||
)
|
||||
assert "duplicate_content" in result["failure_codes"]
|
||||
|
||||
|
||||
def test_conclusion_digest_normalizes_whitespace():
|
||||
assert conclusion_digest("结论:内容 相同") == conclusion_digest("结论:内容 相同\n")
|
||||
assert conclusion_digest("a") != conclusion_digest("b")
|
||||
|
||||
|
||||
def test_repair_requirements_cover_failure_codes():
|
||||
reqs = repair_requirements_for_codes(["missing_evidence", "below_min_length"])
|
||||
assert len(reqs) == 2
|
||||
assert any("证据" in r for r in reqs)
|
||||
|
||||
|
||||
# ── manager review parsing ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_manager_review_valid():
|
||||
model, errors = parse_manager_review("""{"decisions": [
|
||||
{"agent_id": "seat-a", "decision": "return_for_rework",
|
||||
"reason": "缺依据", "revision_requirements": ["补证据"]}
|
||||
]}""")
|
||||
assert errors == []
|
||||
assert model is not None
|
||||
assert model.decisions[0].agent_id == "seat-a"
|
||||
assert model.decisions[0].decision == "return_for_rework"
|
||||
|
||||
|
||||
def test_parse_manager_review_fenced_json():
|
||||
model, errors = parse_manager_review("```json\n{\"decisions\": []}\n```")
|
||||
assert errors == []
|
||||
assert model is not None
|
||||
assert model.decisions == []
|
||||
|
||||
|
||||
def test_parse_manager_review_invalid_decision_value():
|
||||
model, errors = parse_manager_review("""{"decisions": [
|
||||
{"agent_id": "seat-a", "decision": "fake_decision", "reason": "x"}
|
||||
]}""")
|
||||
assert model is None
|
||||
assert errors
|
||||
|
||||
|
||||
def test_parse_manager_review_missing_reason():
|
||||
model, errors = parse_manager_review("""{"decisions": [
|
||||
{"agent_id": "seat-a", "decision": "accept"}
|
||||
]}""")
|
||||
assert model is None
|
||||
assert errors
|
||||
|
||||
|
||||
def test_parse_manager_review_not_json():
|
||||
model, errors = parse_manager_review("definitely not json")
|
||||
assert model is None
|
||||
assert errors
|
||||
|
||||
|
||||
def test_build_manager_review_prompt_contains_contract_and_check():
|
||||
prompt = build_manager_review_prompt(
|
||||
task_text="分析市场",
|
||||
review_items=[{
|
||||
"agent_id": "seat-a",
|
||||
"task_key": "01:seat-a",
|
||||
"attempt": 1,
|
||||
"contract": {"objective": "评估竞品"},
|
||||
"delivery": {"conclusion": "结论"},
|
||||
"check": {"passed": False, "failure_codes": ["missing_evidence"]},
|
||||
}],
|
||||
)
|
||||
assert "评估竞品" in prompt
|
||||
assert "missing_evidence" in prompt
|
||||
assert "seat-a" in prompt
|
||||
|
||||
|
||||
# ── server-owned decision finalization ──────────────────────────────────────
|
||||
|
||||
|
||||
def _contract(**overrides) -> MafTaskContract:
|
||||
base = MafTaskContract(
|
||||
objective="评估竞品",
|
||||
expected_deliverable="结论+证据",
|
||||
acceptance_criteria=["证据"],
|
||||
required=True,
|
||||
max_rework_attempts=2,
|
||||
fallback_policy="reassign_then_gap",
|
||||
)
|
||||
return base.model_copy(update=overrides)
|
||||
|
||||
|
||||
def _item(decision: str, reason: str = "r", requirements: list[str] | None = None) -> MafManagerReviewItem:
|
||||
return MafManagerReviewItem(
|
||||
agent_id="seat-a", decision=decision, reason=reason,
|
||||
revision_requirements=list(requirements or []),
|
||||
)
|
||||
|
||||
|
||||
def _check(passed: bool, codes: list[str] | None = None) -> dict:
|
||||
return {"passed": passed, "failure_codes": list(codes or [])}
|
||||
|
||||
|
||||
def test_finalize_deterministic_fail_never_accepted_even_if_manager_accepts():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(False, ["below_min_length"]),
|
||||
review_item=_item("accept", "看起来可以"),
|
||||
rework_count=0,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REWORK
|
||||
assert final["source"] == "deterministic"
|
||||
assert final["revision_requirements"]
|
||||
|
||||
|
||||
def test_finalize_deterministic_fail_uses_manager_rework_requirements():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(False, ["below_min_length"]),
|
||||
review_item=_item("return_for_rework", "太短", ["请补充具体依据"]),
|
||||
rework_count=0,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REWORK
|
||||
assert final["revision_requirements"] == ["请补充具体依据"]
|
||||
|
||||
|
||||
def test_finalize_deterministic_fail_at_cap_reassigns():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(max_rework_attempts=2),
|
||||
check_result=_check(False, ["below_min_length"]),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REASSIGNED
|
||||
|
||||
|
||||
def test_finalize_deterministic_fail_at_cap_no_backup_controlled_gap():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(False, ["below_min_length"]),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
||||
assert final["source"] == "deterministic"
|
||||
|
||||
|
||||
def test_finalize_deterministic_fail_at_cap_ask_user_fallback():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(fallback_policy="ask_user"),
|
||||
check_result=_check(False, ["below_min_length"]),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_ASK_USER
|
||||
|
||||
|
||||
def test_finalize_pass_manager_accept_accepted():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("accept", "达标"),
|
||||
rework_count=0,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_ACCEPTED
|
||||
assert final["source"] == "manager"
|
||||
|
||||
|
||||
def test_finalize_pass_no_valid_review_never_accepts_forces_rework():
|
||||
# Deterministic checks passed, but the manager review is missing/invalid:
|
||||
# under the rework cap the server forces a system rework — never accepts.
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=None,
|
||||
rework_count=0,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REWORK
|
||||
assert final["source"] == "system"
|
||||
assert final["reason"] == "manager_review_missing_or_invalid"
|
||||
assert final["revision_requirements"]
|
||||
|
||||
|
||||
def test_finalize_pass_no_valid_review_at_cap_follows_fallback():
|
||||
# Rework cap exhausted: the contract fallback policy decides — reassign
|
||||
# first, then controlled_gap, then ask_user. Never accepted.
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REASSIGNED
|
||||
assert final["source"] == "system"
|
||||
assert final["reason"] == "manager_review_missing_or_invalid"
|
||||
|
||||
final_gap = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final_gap["decision"] == OUTCOME_CONTROLLED_GAP
|
||||
assert final_gap["source"] == "system"
|
||||
|
||||
final_ask = finalize_review_decision(
|
||||
contract=_contract(fallback_policy="ask_user"),
|
||||
check_result=_check(True),
|
||||
review_item=None,
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final_ask["decision"] == OUTCOME_ASK_USER
|
||||
assert final_ask["source"] == "system"
|
||||
|
||||
|
||||
def test_finalize_pass_manager_rework_under_cap():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("return_for_rework", "再补充"),
|
||||
rework_count=1,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REWORK
|
||||
|
||||
|
||||
def test_finalize_pass_manager_rework_at_cap_reassigns_then_gap():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("return_for_rework", "仍不足"),
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REASSIGNED
|
||||
|
||||
final2 = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("return_for_rework", "仍不足"),
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final2["decision"] == OUTCOME_CONTROLLED_GAP
|
||||
|
||||
|
||||
def test_finalize_manager_reassign_without_backup_degrades():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("reassign", "换人"),
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
||||
|
||||
|
||||
def test_finalize_manager_reassign_with_backup():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("reassign", "换 seat-c"),
|
||||
rework_count=2,
|
||||
reassign_count=0,
|
||||
has_backup_seat=True,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_REASSIGNED
|
||||
|
||||
|
||||
def test_finalize_manager_ask_user_passthrough():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("ask_user", "需要用户补充"),
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_ASK_USER
|
||||
|
||||
|
||||
def test_finalize_manager_controlled_gap_passthrough():
|
||||
final = finalize_review_decision(
|
||||
contract=_contract(),
|
||||
check_result=_check(True),
|
||||
review_item=_item("controlled_gap", "无法完成"),
|
||||
rework_count=2,
|
||||
reassign_count=1,
|
||||
has_backup_seat=False,
|
||||
)
|
||||
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
||||
assert final["source"] == "manager"
|
||||
|
||||
|
||||
# ── controlled gap records ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_make_controlled_gap_marks_risk():
|
||||
gap = make_controlled_gap(
|
||||
task_key="01:seat-a",
|
||||
agent_id="seat-a",
|
||||
objective="评估竞品",
|
||||
reason="返工上限仍无合格交付",
|
||||
failure_codes=["below_min_length"],
|
||||
attempts=3,
|
||||
)
|
||||
assert gap["task_key"] == "01:seat-a"
|
||||
assert gap["agent_id"] == "seat-a"
|
||||
assert gap["attempts"] == 3
|
||||
assert "不得作为已验证事实" in gap["risk"]
|
||||
|
||||
|
||||
def test_mamager_review_output_model_roundtrip():
|
||||
output = MafManagerReviewOutput(decisions=[
|
||||
{"agent_id": "a", "decision": "accept", "reason": "ok"},
|
||||
])
|
||||
dumped = output.model_dump()
|
||||
assert dumped["decisions"][0]["agent_id"] == "a"
|
||||
@@ -169,6 +169,7 @@ async def test_maf_tables_isolated_from_legacy_roundtable(store):
|
||||
"maf_roundtable_action_leases",
|
||||
"maf_roundtable_approvals",
|
||||
"maf_roundtable_seat_runs",
|
||||
"maf_roundtable_seat_tasks",
|
||||
"maf_roundtable_action_secrets",
|
||||
"maf_roundtable_conversations",
|
||||
"maf_roundtable_messages",
|
||||
|
||||
Reference in new issue
Block a user