"""Stage 4 (R3-Q) pure-policy tests: task contracts, deterministic quality checks, manager review schema, and server-owned decision finalization. These tests exercise ``quality_contracts`` directly (no MAF SDK, no I/O) and prove the weak-model protections: - a delivery that failed deterministic checks can never be accepted, no matter what the manager model claims; - rework is bounded by the frozen contract's ``max_rework_attempts``; - reassignment is bounded by the server-configured cap; - an exhausted task ends as ``controlled_gap`` / ``ask_user``, never as a disguised success. """ from __future__ import annotations from app.maf_roundtable.quality_contracts import ( OUTCOME_ACCEPTED, OUTCOME_ASK_USER, OUTCOME_CONTROLLED_GAP, OUTCOME_REASSIGNED, OUTCOME_REWORK, InvalidTaskContractError, MafManagerReviewItem, MafManagerReviewOutput, MafSeatDelivery, MafTaskContract, build_manager_review_prompt, build_task_contract, check_delivery_deterministic, conclusion_digest, delivery_from_seat_output, finalize_review_decision, make_controlled_gap, parse_manager_review, repair_requirements_for_codes, ) # ── task contract freezing ────────────────────────────────────────────────── def test_build_task_contract_freezes_from_dispatch_assignment(): contract = build_task_contract( task_goal="评估竞品定价策略", delivery_requirements=["风险清单", "依据", "优先级", "建议"], required=True, max_rework_attempts=2, fallback_policy="reassign_then_gap", ) assert contract.objective == "评估竞品定价策略" assert contract.expected_deliverable assert contract.acceptance_criteria == ["风险清单", "依据", "优先级", "建议"] assert contract.required is True assert contract.max_rework_attempts == 2 assert contract.fallback_policy == "reassign_then_gap" def test_build_task_contract_rejects_empty_objective(): try: build_task_contract(task_goal=" ", delivery_requirements=[]) except InvalidTaskContractError: return raise AssertionError("empty objective must raise InvalidTaskContractError") def test_build_task_contract_strips_blank_criteria(): contract = build_task_contract(task_goal="t", delivery_requirements=["a", " ", "b"]) assert contract.acceptance_criteria == ["a", "b"] # ── delivery mapping ──────────────────────────────────────────────────────── def test_delivery_from_seat_output_maps_all_fields(): delivery = delivery_from_seat_output( structured_output={ "summary": "结论:竞品定价偏高", "findings": ["find-1", {"text": "find-2"}], "evidence_refs": [{"ref": "ref-a"}, "ref-b"], "open_questions": ["q1"], "self_check": "已覆盖优先级标准", }, raw_content='{"summary": "结论:竞品定价偏高"}', ) assert delivery.conclusion == "结论:竞品定价偏高" assert delivery.evidence == ["find-1", "find-2", "ref-a", "ref-b"] # P1: only evidence_refs can satisfy the evidence requirement. assert delivery.evidence_refs == ["ref-a", "ref-b"] assert delivery.unresolved_items == ["q1"] assert delivery.self_check == "已覆盖优先级标准" def test_delivery_from_seat_output_schema_invalid_is_empty(): delivery = delivery_from_seat_output(structured_output=None, raw_content="not json") assert delivery.conclusion == "" assert delivery.evidence == [] assert delivery.evidence_refs == [] # ── deterministic checks ──────────────────────────────────────────────────── def _good_delivery() -> MafSeatDelivery: return MafSeatDelivery( conclusion="结论:该市场存在明确增长机会,建议优先进入华东区域并配套本地化运营资源投入,同时预留华南试点预算", evidence=["ref-1", "ref-2"], evidence_refs=["ref-1", "ref-2"], unresolved_items=[], self_check="ok", ) def test_check_delivery_deterministic_passes_well_formed(): delivery = _good_delivery() result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", ) assert result["passed"] is True assert result["failure_codes"] == [] def test_check_delivery_deterministic_empty_content(): result = check_delivery_deterministic( delivery=_good_delivery(), raw_content=" ", schema_status="ok", ) assert result["passed"] is False assert "empty_content" in result["failure_codes"] def test_check_delivery_deterministic_schema_invalid(): result = check_delivery_deterministic( delivery=_good_delivery(), raw_content="not json", schema_status="schema_invalid", ) assert result["passed"] is False assert "schema_invalid" in result["failure_codes"] def test_check_delivery_deterministic_missing_conclusion(): delivery = MafSeatDelivery(conclusion="", evidence=["e"], self_check="") result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":""}', schema_status="ok", ) assert "missing_conclusion" in result["failure_codes"] def test_check_delivery_deterministic_below_min_length(): delivery = MafSeatDelivery(conclusion="好", evidence=["e"], self_check="") result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"好"}', schema_status="ok", min_length=40, ) assert "below_min_length" in result["failure_codes"] def test_check_delivery_deterministic_missing_evidence(): delivery = MafSeatDelivery(conclusion="这是一个足够长的结论用于通过最小长度校验", evidence=[], self_check="") result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1, ) assert "missing_evidence" in result["failure_codes"] def test_check_delivery_findings_alone_never_satisfy_evidence(): # P1: findings are candidate claims only — a delivery full of findings but # without evidence_refs still fails the evidence requirement. delivery = MafSeatDelivery( conclusion="这是一个足够长的结论用于通过最小长度校验,附带大量候选主张作为干扰项", evidence=["我查到 A", "我查到 B", "find-3"], evidence_refs=[], self_check="", ) result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1, ) assert result["passed"] is False assert "missing_evidence" in result["failure_codes"] def test_check_delivery_invented_evidence_ref_fails_unverified(): # P1: a cited ref that the fact ledger does not back is a hallucination. delivery = MafSeatDelivery( conclusion="这是一个足够长的结论用于通过最小长度校验,引用了不存在的证据", evidence=["fake-ref-001"], evidence_refs=["fake-ref-001"], self_check="", ) result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1, verified_evidence_refs={"tool:kb_search:1", "ref-real"}, ) assert result["passed"] is False assert "unverified_evidence_ref" in result["failure_codes"] # The invented ref also cannot count toward the evidence minimum. assert "missing_evidence" in result["failure_codes"] def test_check_delivery_verified_evidence_ref_passes(): delivery = MafSeatDelivery( conclusion="这是一个足够长的结论用于通过最小长度校验,引用了事实账本中真实存在的证据引用,满足最小证据数量要求", evidence=["tool:kb_search:1"], evidence_refs=["tool:kb_search:1"], self_check="", ) result = check_delivery_deterministic( delivery=delivery, raw_content='{"summary":"x"}', schema_status="ok", min_evidence=1, verified_evidence_refs={"tool:kb_search:1"}, ) assert result["passed"] is True assert result["failure_codes"] == [] def test_check_delivery_deterministic_duplicate_content_by_digest(): delivery = MafSeatDelivery( conclusion="结论:与上次完全相同的内容", evidence=["e"], evidence_refs=["e"], self_check="", ) result = check_delivery_deterministic( delivery=delivery, raw_content="x", schema_status="ok", prior_conclusion_digests={conclusion_digest("结论:与上次完全相同的内容")}, ) assert "duplicate_content" in result["failure_codes"] def test_conclusion_digest_normalizes_whitespace(): assert conclusion_digest("结论:内容 相同") == conclusion_digest("结论:内容 相同\n") assert conclusion_digest("a") != conclusion_digest("b") def test_repair_requirements_cover_failure_codes(): reqs = repair_requirements_for_codes(["missing_evidence", "below_min_length"]) assert len(reqs) == 2 assert any("证据" in r for r in reqs) # ── manager review parsing ────────────────────────────────────────────────── def test_parse_manager_review_valid(): model, errors = parse_manager_review("""{"decisions": [ {"agent_id": "seat-a", "decision": "return_for_rework", "reason": "缺依据", "revision_requirements": ["补证据"]} ]}""") assert errors == [] assert model is not None assert model.decisions[0].agent_id == "seat-a" assert model.decisions[0].decision == "return_for_rework" def test_parse_manager_review_fenced_json(): model, errors = parse_manager_review("```json\n{\"decisions\": []}\n```") assert errors == [] assert model is not None assert model.decisions == [] def test_parse_manager_review_invalid_decision_value(): model, errors = parse_manager_review("""{"decisions": [ {"agent_id": "seat-a", "decision": "fake_decision", "reason": "x"} ]}""") assert model is None assert errors def test_parse_manager_review_missing_reason(): model, errors = parse_manager_review("""{"decisions": [ {"agent_id": "seat-a", "decision": "accept"} ]}""") assert model is None assert errors def test_parse_manager_review_not_json(): model, errors = parse_manager_review("definitely not json") assert model is None assert errors def test_build_manager_review_prompt_contains_contract_and_check(): prompt = build_manager_review_prompt( task_text="分析市场", review_items=[{ "agent_id": "seat-a", "task_key": "01:seat-a", "attempt": 1, "contract": {"objective": "评估竞品"}, "delivery": {"conclusion": "结论"}, "check": {"passed": False, "failure_codes": ["missing_evidence"]}, }], ) assert "评估竞品" in prompt assert "missing_evidence" in prompt assert "seat-a" in prompt # ── server-owned decision finalization ────────────────────────────────────── def _contract(**overrides) -> MafTaskContract: base = MafTaskContract( objective="评估竞品", expected_deliverable="结论+证据", acceptance_criteria=["证据"], required=True, max_rework_attempts=2, fallback_policy="reassign_then_gap", ) return base.model_copy(update=overrides) def _item(decision: str, reason: str = "r", requirements: list[str] | None = None) -> MafManagerReviewItem: return MafManagerReviewItem( agent_id="seat-a", decision=decision, reason=reason, revision_requirements=list(requirements or []), ) def _check(passed: bool, codes: list[str] | None = None) -> dict: return {"passed": passed, "failure_codes": list(codes or [])} def test_finalize_deterministic_fail_never_accepted_even_if_manager_accepts(): final = finalize_review_decision( contract=_contract(), check_result=_check(False, ["below_min_length"]), review_item=_item("accept", "看起来可以"), rework_count=0, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REWORK assert final["source"] == "deterministic" assert final["revision_requirements"] def test_finalize_deterministic_fail_uses_manager_rework_requirements(): final = finalize_review_decision( contract=_contract(), check_result=_check(False, ["below_min_length"]), review_item=_item("return_for_rework", "太短", ["请补充具体依据"]), rework_count=0, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REWORK assert final["revision_requirements"] == ["请补充具体依据"] def test_finalize_deterministic_fail_at_cap_reassigns(): final = finalize_review_decision( contract=_contract(max_rework_attempts=2), check_result=_check(False, ["below_min_length"]), review_item=None, rework_count=2, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REASSIGNED def test_finalize_deterministic_fail_at_cap_no_backup_controlled_gap(): final = finalize_review_decision( contract=_contract(), check_result=_check(False, ["below_min_length"]), review_item=None, rework_count=2, reassign_count=0, has_backup_seat=False, ) assert final["decision"] == OUTCOME_CONTROLLED_GAP assert final["source"] == "deterministic" def test_finalize_deterministic_fail_at_cap_ask_user_fallback(): final = finalize_review_decision( contract=_contract(fallback_policy="ask_user"), check_result=_check(False, ["below_min_length"]), review_item=None, rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final["decision"] == OUTCOME_ASK_USER def test_finalize_pass_manager_accept_accepted(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("accept", "达标"), rework_count=0, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_ACCEPTED assert final["source"] == "manager" def test_finalize_pass_no_valid_review_never_accepts_forces_rework(): # Deterministic checks passed, but the manager review is missing/invalid: # under the rework cap the server forces a system rework — never accepts. final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=None, rework_count=0, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REWORK assert final["source"] == "system" assert final["reason"] == "manager_review_missing_or_invalid" assert final["revision_requirements"] def test_finalize_pass_no_valid_review_at_cap_follows_fallback(): # Rework cap exhausted: the contract fallback policy decides — reassign # first, then controlled_gap, then ask_user. Never accepted. final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=None, rework_count=2, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REASSIGNED assert final["source"] == "system" assert final["reason"] == "manager_review_missing_or_invalid" final_gap = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=None, rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final_gap["decision"] == OUTCOME_CONTROLLED_GAP assert final_gap["source"] == "system" final_ask = finalize_review_decision( contract=_contract(fallback_policy="ask_user"), check_result=_check(True), review_item=None, rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final_ask["decision"] == OUTCOME_ASK_USER assert final_ask["source"] == "system" def test_finalize_pass_manager_rework_under_cap(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("return_for_rework", "再补充"), rework_count=1, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REWORK def test_finalize_pass_manager_rework_at_cap_reassigns_then_gap(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("return_for_rework", "仍不足"), rework_count=2, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REASSIGNED final2 = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("return_for_rework", "仍不足"), rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final2["decision"] == OUTCOME_CONTROLLED_GAP def test_finalize_manager_reassign_without_backup_degrades(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("reassign", "换人"), rework_count=2, reassign_count=0, has_backup_seat=False, ) assert final["decision"] == OUTCOME_CONTROLLED_GAP def test_finalize_manager_reassign_with_backup(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("reassign", "换 seat-c"), rework_count=2, reassign_count=0, has_backup_seat=True, ) assert final["decision"] == OUTCOME_REASSIGNED def test_finalize_manager_ask_user_passthrough(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("ask_user", "需要用户补充"), rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final["decision"] == OUTCOME_ASK_USER def test_finalize_manager_controlled_gap_passthrough(): final = finalize_review_decision( contract=_contract(), check_result=_check(True), review_item=_item("controlled_gap", "无法完成"), rework_count=2, reassign_count=1, has_backup_seat=False, ) assert final["decision"] == OUTCOME_CONTROLLED_GAP assert final["source"] == "manager" # ── controlled gap records ────────────────────────────────────────────────── def test_make_controlled_gap_marks_risk(): gap = make_controlled_gap( task_key="01:seat-a", agent_id="seat-a", objective="评估竞品", reason="返工上限仍无合格交付", failure_codes=["below_min_length"], attempts=3, ) assert gap["task_key"] == "01:seat-a" assert gap["agent_id"] == "seat-a" assert gap["attempts"] == 3 assert "不得作为已验证事实" in gap["risk"] def test_mamager_review_output_model_roundtrip(): output = MafManagerReviewOutput(decisions=[ {"agent_id": "a", "decision": "accept", "reason": "ok"}, ]) dumped = output.model_dump() assert dumped["decisions"][0]["agent_id"] == "a"