董宏凯
1aeee86984
P0-1 migration: detect the same-named stage-3 execution_stage CHECK by sqltext (must contain rework + manager_review), SQLite rebuild / PG-MySQL drop+add, PG boolean default TRUE, validate_schema checks constraint content + seat_tasks status CHECK/unique/FKs; tests start from a real stage-3 schema. P0-2: a missing/invalid manager review (timeout/exception/empty/invalid JSON/missing per-seat decision) is never an acceptance - system rework under the cap, contract fallback (reassign/controlled_gap/ask_user) afterwards, source=system events. P0-3: unaccepted deliveries are isolated from the fact ledger, cross-review and final report (deferred merge, delivery_json envelope, accepted_task_results in checkpoint, accepted-only synthesis, no turn_results fallback). P0-4: seat-task init is atomic get-or-create over the full expected set; recovery fills missing tasks without overwriting frozen contracts; quality gate blocks missing required tasks. P1: zero config values honoured; duplicate detection persists SHA-256 conclusion digests in history (survives restarts); findings never satisfy evidence, evidence_refs must resolve to trusted ledger refs (unverified_evidence_ref); conversation SSE projection tests added and manager/finalizer messages are no longer mislabelled as seat.delivery.submitted.
566 lines
20 KiB
Python
566 lines
20 KiB
Python
"""Stage 4 (R3-Q) pure-policy tests: task contracts, deterministic quality
|
|
checks, manager review schema, and server-owned decision finalization.
|
|
|
|
These tests exercise ``quality_contracts`` directly (no MAF SDK, no I/O) and
|
|
prove the weak-model protections:
|
|
|
|
- a delivery that failed deterministic checks can never be accepted, no matter
|
|
what the manager model claims;
|
|
- rework is bounded by the frozen contract's ``max_rework_attempts``;
|
|
- reassignment is bounded by the server-configured cap;
|
|
- an exhausted task ends as ``controlled_gap`` / ``ask_user``, never as a
|
|
disguised success.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.maf_roundtable.quality_contracts import (
|
|
OUTCOME_ACCEPTED,
|
|
OUTCOME_ASK_USER,
|
|
OUTCOME_CONTROLLED_GAP,
|
|
OUTCOME_REASSIGNED,
|
|
OUTCOME_REWORK,
|
|
InvalidTaskContractError,
|
|
MafManagerReviewItem,
|
|
MafManagerReviewOutput,
|
|
MafSeatDelivery,
|
|
MafTaskContract,
|
|
build_manager_review_prompt,
|
|
build_task_contract,
|
|
check_delivery_deterministic,
|
|
conclusion_digest,
|
|
delivery_from_seat_output,
|
|
finalize_review_decision,
|
|
make_controlled_gap,
|
|
parse_manager_review,
|
|
repair_requirements_for_codes,
|
|
)
|
|
|
|
# ── task contract freezing ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_build_task_contract_freezes_from_dispatch_assignment():
|
|
contract = build_task_contract(
|
|
task_goal="评估竞品定价策略",
|
|
delivery_requirements=["风险清单", "依据", "优先级", "建议"],
|
|
required=True,
|
|
max_rework_attempts=2,
|
|
fallback_policy="reassign_then_gap",
|
|
)
|
|
assert contract.objective == "评估竞品定价策略"
|
|
assert contract.expected_deliverable
|
|
assert contract.acceptance_criteria == ["风险清单", "依据", "优先级", "建议"]
|
|
assert contract.required is True
|
|
assert contract.max_rework_attempts == 2
|
|
assert contract.fallback_policy == "reassign_then_gap"
|
|
|
|
|
|
def test_build_task_contract_rejects_empty_objective():
|
|
try:
|
|
build_task_contract(task_goal=" ", delivery_requirements=[])
|
|
except InvalidTaskContractError:
|
|
return
|
|
raise AssertionError("empty objective must raise InvalidTaskContractError")
|
|
|
|
|
|
def test_build_task_contract_strips_blank_criteria():
|
|
contract = build_task_contract(task_goal="t", delivery_requirements=["a", " ", "b"])
|
|
assert contract.acceptance_criteria == ["a", "b"]
|
|
|
|
|
|
# ── delivery mapping ────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_delivery_from_seat_output_maps_all_fields():
|
|
delivery = delivery_from_seat_output(
|
|
structured_output={
|
|
"summary": "结论：竞品定价偏高",
|
|
"findings": ["find-1", {"text": "find-2"}],
|
|
"evidence_refs": [{"ref": "ref-a"}, "ref-b"],
|
|
"open_questions": ["q1"],
|
|
"self_check": "已覆盖优先级标准",
|
|
},
|
|
raw_content='{"summary": "结论：竞品定价偏高"}',
|
|
)
|
|
assert delivery.conclusion == "结论：竞品定价偏高"
|
|
assert delivery.evidence == ["find-1", "find-2", "ref-a", "ref-b"]
|
|
# P1: only evidence_refs can satisfy the evidence requirement.
|
|
assert delivery.evidence_refs == ["ref-a", "ref-b"]
|
|
assert delivery.unresolved_items == ["q1"]
|
|
assert delivery.self_check == "已覆盖优先级标准"
|
|
|
|
|
|
def test_delivery_from_seat_output_schema_invalid_is_empty():
|
|
delivery = delivery_from_seat_output(structured_output=None, raw_content="not json")
|
|
assert delivery.conclusion == ""
|
|
assert delivery.evidence == []
|
|
assert delivery.evidence_refs == []
|
|
|
|
|
|
# ── deterministic checks ────────────────────────────────────────────────────
|
|
|
|
|
|
def _good_delivery() -> MafSeatDelivery:
|
|
return MafSeatDelivery(
|
|
conclusion="结论：该市场存在明确增长机会，建议优先进入华东区域并配套本地化运营资源投入，同时预留华南试点预算",
|
|
evidence=["ref-1", "ref-2"],
|
|
evidence_refs=["ref-1", "ref-2"],
|
|
unresolved_items=[],
|
|
self_check="ok",
|
|
)
|
|
|
|
|
|
def test_check_delivery_deterministic_passes_well_formed():
|
|
delivery = _good_delivery()
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok",
|
|
)
|
|
assert result["passed"] is True
|
|
assert result["failure_codes"] == []
|
|
|
|
|
|
def test_check_delivery_deterministic_empty_content():
|
|
result = check_delivery_deterministic(
|
|
delivery=_good_delivery(), raw_content=" ", schema_status="ok",
|
|
)
|
|
assert result["passed"] is False
|
|
assert "empty_content" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_deterministic_schema_invalid():
|
|
result = check_delivery_deterministic(
|
|
delivery=_good_delivery(), raw_content="not json", schema_status="schema_invalid",
|
|
)
|
|
assert result["passed"] is False
|
|
assert "schema_invalid" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_deterministic_missing_conclusion():
|
|
delivery = MafSeatDelivery(conclusion="", evidence=["e"], self_check="")
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery, raw_content='{"summary":""}', schema_status="ok",
|
|
)
|
|
assert "missing_conclusion" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_deterministic_below_min_length():
|
|
delivery = MafSeatDelivery(conclusion="好", evidence=["e"], self_check="")
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery, raw_content='{"summary":"好"}', schema_status="ok", min_length=40,
|
|
)
|
|
assert "below_min_length" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_deterministic_missing_evidence():
|
|
delivery = MafSeatDelivery(conclusion="这是一个足够长的结论用于通过最小长度校验", evidence=[], self_check="")
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1,
|
|
)
|
|
assert "missing_evidence" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_findings_alone_never_satisfy_evidence():
|
|
# P1: findings are candidate claims only — a delivery full of findings but
|
|
# without evidence_refs still fails the evidence requirement.
|
|
delivery = MafSeatDelivery(
|
|
conclusion="这是一个足够长的结论用于通过最小长度校验，附带大量候选主张作为干扰项",
|
|
evidence=["我查到 A", "我查到 B", "find-3"],
|
|
evidence_refs=[],
|
|
self_check="",
|
|
)
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1,
|
|
)
|
|
assert result["passed"] is False
|
|
assert "missing_evidence" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_invented_evidence_ref_fails_unverified():
|
|
# P1: a cited ref that the fact ledger does not back is a hallucination.
|
|
delivery = MafSeatDelivery(
|
|
conclusion="这是一个足够长的结论用于通过最小长度校验，引用了不存在的证据",
|
|
evidence=["fake-ref-001"],
|
|
evidence_refs=["fake-ref-001"],
|
|
self_check="",
|
|
)
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery,
|
|
raw_content='{"summary":"x"}',
|
|
schema_status="ok",
|
|
min_evidence=1,
|
|
verified_evidence_refs={"tool:kb_search:1", "ref-real"},
|
|
)
|
|
assert result["passed"] is False
|
|
assert "unverified_evidence_ref" in result["failure_codes"]
|
|
# The invented ref also cannot count toward the evidence minimum.
|
|
assert "missing_evidence" in result["failure_codes"]
|
|
|
|
|
|
def test_check_delivery_verified_evidence_ref_passes():
|
|
delivery = MafSeatDelivery(
|
|
conclusion="这是一个足够长的结论用于通过最小长度校验，引用了事实账本中真实存在的证据引用，满足最小证据数量要求",
|
|
evidence=["tool:kb_search:1"],
|
|
evidence_refs=["tool:kb_search:1"],
|
|
self_check="",
|
|
)
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery,
|
|
raw_content='{"summary":"x"}',
|
|
schema_status="ok",
|
|
min_evidence=1,
|
|
verified_evidence_refs={"tool:kb_search:1"},
|
|
)
|
|
assert result["passed"] is True
|
|
assert result["failure_codes"] == []
|
|
|
|
|
|
def test_check_delivery_deterministic_duplicate_content_by_digest():
|
|
delivery = MafSeatDelivery(
|
|
conclusion="结论：与上次完全相同的内容", evidence=["e"], evidence_refs=["e"], self_check="",
|
|
)
|
|
result = check_delivery_deterministic(
|
|
delivery=delivery,
|
|
raw_content="x",
|
|
schema_status="ok",
|
|
prior_conclusion_digests={conclusion_digest("结论：与上次完全相同的内容")},
|
|
)
|
|
assert "duplicate_content" in result["failure_codes"]
|
|
|
|
|
|
def test_conclusion_digest_normalizes_whitespace():
|
|
assert conclusion_digest("结论：内容 相同") == conclusion_digest("结论：内容 相同\n")
|
|
assert conclusion_digest("a") != conclusion_digest("b")
|
|
|
|
|
|
def test_repair_requirements_cover_failure_codes():
|
|
reqs = repair_requirements_for_codes(["missing_evidence", "below_min_length"])
|
|
assert len(reqs) == 2
|
|
assert any("证据" in r for r in reqs)
|
|
|
|
|
|
# ── manager review parsing ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_parse_manager_review_valid():
|
|
model, errors = parse_manager_review("""{"decisions": [
|
|
{"agent_id": "seat-a", "decision": "return_for_rework",
|
|
"reason": "缺依据", "revision_requirements": ["补证据"]}
|
|
]}""")
|
|
assert errors == []
|
|
assert model is not None
|
|
assert model.decisions[0].agent_id == "seat-a"
|
|
assert model.decisions[0].decision == "return_for_rework"
|
|
|
|
|
|
def test_parse_manager_review_fenced_json():
|
|
model, errors = parse_manager_review("```json\n{\"decisions\": []}\n```")
|
|
assert errors == []
|
|
assert model is not None
|
|
assert model.decisions == []
|
|
|
|
|
|
def test_parse_manager_review_invalid_decision_value():
|
|
model, errors = parse_manager_review("""{"decisions": [
|
|
{"agent_id": "seat-a", "decision": "fake_decision", "reason": "x"}
|
|
]}""")
|
|
assert model is None
|
|
assert errors
|
|
|
|
|
|
def test_parse_manager_review_missing_reason():
|
|
model, errors = parse_manager_review("""{"decisions": [
|
|
{"agent_id": "seat-a", "decision": "accept"}
|
|
]}""")
|
|
assert model is None
|
|
assert errors
|
|
|
|
|
|
def test_parse_manager_review_not_json():
|
|
model, errors = parse_manager_review("definitely not json")
|
|
assert model is None
|
|
assert errors
|
|
|
|
|
|
def test_build_manager_review_prompt_contains_contract_and_check():
|
|
prompt = build_manager_review_prompt(
|
|
task_text="分析市场",
|
|
review_items=[{
|
|
"agent_id": "seat-a",
|
|
"task_key": "01:seat-a",
|
|
"attempt": 1,
|
|
"contract": {"objective": "评估竞品"},
|
|
"delivery": {"conclusion": "结论"},
|
|
"check": {"passed": False, "failure_codes": ["missing_evidence"]},
|
|
}],
|
|
)
|
|
assert "评估竞品" in prompt
|
|
assert "missing_evidence" in prompt
|
|
assert "seat-a" in prompt
|
|
|
|
|
|
# ── server-owned decision finalization ──────────────────────────────────────
|
|
|
|
|
|
def _contract(**overrides) -> MafTaskContract:
|
|
base = MafTaskContract(
|
|
objective="评估竞品",
|
|
expected_deliverable="结论+证据",
|
|
acceptance_criteria=["证据"],
|
|
required=True,
|
|
max_rework_attempts=2,
|
|
fallback_policy="reassign_then_gap",
|
|
)
|
|
return base.model_copy(update=overrides)
|
|
|
|
|
|
def _item(decision: str, reason: str = "r", requirements: list[str] | None = None) -> MafManagerReviewItem:
|
|
return MafManagerReviewItem(
|
|
agent_id="seat-a", decision=decision, reason=reason,
|
|
revision_requirements=list(requirements or []),
|
|
)
|
|
|
|
|
|
def _check(passed: bool, codes: list[str] | None = None) -> dict:
|
|
return {"passed": passed, "failure_codes": list(codes or [])}
|
|
|
|
|
|
def test_finalize_deterministic_fail_never_accepted_even_if_manager_accepts():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(False, ["below_min_length"]),
|
|
review_item=_item("accept", "看起来可以"),
|
|
rework_count=0,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REWORK
|
|
assert final["source"] == "deterministic"
|
|
assert final["revision_requirements"]
|
|
|
|
|
|
def test_finalize_deterministic_fail_uses_manager_rework_requirements():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(False, ["below_min_length"]),
|
|
review_item=_item("return_for_rework", "太短", ["请补充具体依据"]),
|
|
rework_count=0,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REWORK
|
|
assert final["revision_requirements"] == ["请补充具体依据"]
|
|
|
|
|
|
def test_finalize_deterministic_fail_at_cap_reassigns():
|
|
final = finalize_review_decision(
|
|
contract=_contract(max_rework_attempts=2),
|
|
check_result=_check(False, ["below_min_length"]),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REASSIGNED
|
|
|
|
|
|
def test_finalize_deterministic_fail_at_cap_no_backup_controlled_gap():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(False, ["below_min_length"]),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
|
assert final["source"] == "deterministic"
|
|
|
|
|
|
def test_finalize_deterministic_fail_at_cap_ask_user_fallback():
|
|
final = finalize_review_decision(
|
|
contract=_contract(fallback_policy="ask_user"),
|
|
check_result=_check(False, ["below_min_length"]),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final["decision"] == OUTCOME_ASK_USER
|
|
|
|
|
|
def test_finalize_pass_manager_accept_accepted():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("accept", "达标"),
|
|
rework_count=0,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_ACCEPTED
|
|
assert final["source"] == "manager"
|
|
|
|
|
|
def test_finalize_pass_no_valid_review_never_accepts_forces_rework():
|
|
# Deterministic checks passed, but the manager review is missing/invalid:
|
|
# under the rework cap the server forces a system rework — never accepts.
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=None,
|
|
rework_count=0,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REWORK
|
|
assert final["source"] == "system"
|
|
assert final["reason"] == "manager_review_missing_or_invalid"
|
|
assert final["revision_requirements"]
|
|
|
|
|
|
def test_finalize_pass_no_valid_review_at_cap_follows_fallback():
|
|
# Rework cap exhausted: the contract fallback policy decides — reassign
|
|
# first, then controlled_gap, then ask_user. Never accepted.
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REASSIGNED
|
|
assert final["source"] == "system"
|
|
assert final["reason"] == "manager_review_missing_or_invalid"
|
|
|
|
final_gap = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final_gap["decision"] == OUTCOME_CONTROLLED_GAP
|
|
assert final_gap["source"] == "system"
|
|
|
|
final_ask = finalize_review_decision(
|
|
contract=_contract(fallback_policy="ask_user"),
|
|
check_result=_check(True),
|
|
review_item=None,
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final_ask["decision"] == OUTCOME_ASK_USER
|
|
assert final_ask["source"] == "system"
|
|
|
|
|
|
def test_finalize_pass_manager_rework_under_cap():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("return_for_rework", "再补充"),
|
|
rework_count=1,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REWORK
|
|
|
|
|
|
def test_finalize_pass_manager_rework_at_cap_reassigns_then_gap():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("return_for_rework", "仍不足"),
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REASSIGNED
|
|
|
|
final2 = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("return_for_rework", "仍不足"),
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final2["decision"] == OUTCOME_CONTROLLED_GAP
|
|
|
|
|
|
def test_finalize_manager_reassign_without_backup_degrades():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("reassign", "换人"),
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
|
|
|
|
|
def test_finalize_manager_reassign_with_backup():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("reassign", "换 seat-c"),
|
|
rework_count=2,
|
|
reassign_count=0,
|
|
has_backup_seat=True,
|
|
)
|
|
assert final["decision"] == OUTCOME_REASSIGNED
|
|
|
|
|
|
def test_finalize_manager_ask_user_passthrough():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("ask_user", "需要用户补充"),
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final["decision"] == OUTCOME_ASK_USER
|
|
|
|
|
|
def test_finalize_manager_controlled_gap_passthrough():
|
|
final = finalize_review_decision(
|
|
contract=_contract(),
|
|
check_result=_check(True),
|
|
review_item=_item("controlled_gap", "无法完成"),
|
|
rework_count=2,
|
|
reassign_count=1,
|
|
has_backup_seat=False,
|
|
)
|
|
assert final["decision"] == OUTCOME_CONTROLLED_GAP
|
|
assert final["source"] == "manager"
|
|
|
|
|
|
# ── controlled gap records ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_make_controlled_gap_marks_risk():
|
|
gap = make_controlled_gap(
|
|
task_key="01:seat-a",
|
|
agent_id="seat-a",
|
|
objective="评估竞品",
|
|
reason="返工上限仍无合格交付",
|
|
failure_codes=["below_min_length"],
|
|
attempts=3,
|
|
)
|
|
assert gap["task_key"] == "01:seat-a"
|
|
assert gap["agent_id"] == "seat-a"
|
|
assert gap["attempts"] == 3
|
|
assert "不得作为已验证事实" in gap["risk"]
|
|
|
|
|
|
def test_mamager_review_output_model_roundtrip():
|
|
output = MafManagerReviewOutput(decisions=[
|
|
{"agent_id": "a", "decision": "accept", "reason": "ok"},
|
|
])
|
|
dumped = output.model_dump()
|
|
assert dumped["decisions"][0]["agent_id"] == "a"
|