初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,219 @@
|
||||
"""RC-BE-018: 20 annotated tasks, deterministic scoring, blind compare, reliability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
import pytest
|
||||
|
||||
from app.report_collaboration.eval import (
|
||||
METRIC_GATES,
|
||||
RECOMMENDED_RUNTIME_PROFILE,
|
||||
compare_blind,
|
||||
evaluate_catalog,
|
||||
evaluate_task,
|
||||
get_eval_task,
|
||||
load_eval_tasks,
|
||||
materialize_failure,
|
||||
materialize_gold,
|
||||
score_report,
|
||||
summarize_reliability,
|
||||
)
|
||||
from app.report_collaboration.eval.schema import ReportCandidate
|
||||
from app.report_collaboration.eval.catalog import STANDARD_DIRECTED_MODS
|
||||
from app.report_collaboration.execution.ledger import TaskResult
|
||||
from app.report_collaboration.execution.member_runner import WaveExecutor
|
||||
from app.report_collaboration.execution.quality_roles import QualityRoleKernel
|
||||
from app.report_collaboration.interventions.intent_router import RuntimeIntentContext, RuntimeIntentRouterAgent
|
||||
from deerflow.config.report_collaboration_config import ReportCollaborationConfig
|
||||
from deerflow.persistence.report_collaboration import MemoryReportCollaborationStore
|
||||
|
||||
|
||||
def test_catalog_has_twenty_balanced_tasks() -> None:
|
||||
tasks = load_eval_tasks()
|
||||
assert len(tasks) == 20
|
||||
assert len({item.id for item in tasks}) == 20
|
||||
counts = Counter(item.category for item in tasks)
|
||||
assert counts == {"policy": 4, "market": 4, "enterprise": 4, "event": 4, "comparison": 4}
|
||||
assert get_eval_task("ev-05").title == "新能源汽车主要品牌市场趋势"
|
||||
for task in tasks:
|
||||
assert task.required_angles
|
||||
assert task.key_facts
|
||||
assert task.forbidden_fabrications
|
||||
assert task.template.sections
|
||||
assert task.review_rules
|
||||
assert task.acceptable_sources
|
||||
assert [item.id for item in task.directed_modifications] == [item.id for item in STANDARD_DIRECTED_MODS]
|
||||
|
||||
|
||||
def test_recommended_profile_matches_disabled_production_defaults() -> None:
|
||||
config = ReportCollaborationConfig()
|
||||
for key, value in RECOMMENDED_RUNTIME_PROFILE.items():
|
||||
assert getattr(config, key) == value
|
||||
assert config.worker_enabled is False
|
||||
assert METRIC_GATES["citation_coverage"] >= 0.85
|
||||
assert METRIC_GATES["angle_coverage"] == 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_gold_reports_clear_eval_gates() -> None:
|
||||
summary = await evaluate_catalog()
|
||||
assert summary.tasks == 20
|
||||
assert summary.passed == 20
|
||||
assert summary.composite >= 0.95
|
||||
assert set(summary.by_category) == {"policy", "market", "enterprise", "event", "comparison"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_known_failures_are_reproducible() -> None:
|
||||
task = get_eval_task("ev-05")
|
||||
gold = await evaluate_task(task, materialize_gold(task))
|
||||
assert gold.passed
|
||||
fabricated = await evaluate_task(task, materialize_failure(task, "fabricated"))
|
||||
assert not fabricated.passed
|
||||
assert not fabricated.metric("factual_accuracy").passed
|
||||
missing = await evaluate_task(task, materialize_failure(task, "missing_angle"))
|
||||
assert not missing.metric("angle_coverage").passed
|
||||
uncited = await evaluate_task(task, materialize_failure(task, "uncited"))
|
||||
assert not uncited.metric("citation_coverage").passed
|
||||
empty = score_report(task, materialize_failure(task, "empty"), directed_hits=[True, True, True])
|
||||
assert not empty.passed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blind_compare_keeps_roundtable_failure_instance() -> None:
|
||||
task = get_eval_task("ev-05")
|
||||
hits = [True, True, True]
|
||||
result = compare_blind(task, materialize_gold(task), materialize_failure(task, "roundtable_essay"), directed_hits=hits)
|
||||
assert result.winner == "collaboration"
|
||||
assert result.deltas["citation_coverage"] > 0
|
||||
assert result.deltas["template_adherence"] > 0
|
||||
assert not result.roundtable.passed
|
||||
assert "不得仅以" in result.notes
|
||||
|
||||
|
||||
def test_reliability_counts_twenty_fixed_runs() -> None:
|
||||
task = get_eval_task("ev-05")
|
||||
golds = [materialize_gold(task) for _ in range(17)]
|
||||
failures = [
|
||||
materialize_failure(task, "empty"),
|
||||
materialize_failure(task, "illegal_json"),
|
||||
materialize_gold(task).model_copy(update={"status": "failed", "notes": "early_stop"}),
|
||||
]
|
||||
counts = summarize_reliability(golds + failures)
|
||||
assert counts.runs == 20
|
||||
assert counts.completed == 17
|
||||
assert counts.empty_output == 1
|
||||
assert counts.illegal_json == 1
|
||||
assert counts.early_stop == 1
|
||||
assert counts.failure_rate == pytest.approx(3 / 20)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rewrite_without_research_is_not_classified_as_research_again() -> None:
|
||||
task = get_eval_task("ev-05")
|
||||
router = RuntimeIntentRouterAgent()
|
||||
rewrite = await router.route(
|
||||
RuntimeIntentContext(
|
||||
text="重写第二章,结论更谨慎,不要重新检索",
|
||||
requirement={"topic": task.title, "required_angles": task.required_angles, "revision": 1},
|
||||
plan_nodes=[{"id": "write", "role_key": "writer"}, {"id": "research", "role_key": "researcher"}],
|
||||
)
|
||||
)
|
||||
assert rewrite.intent == "rewrite_section"
|
||||
research = await router.route(
|
||||
RuntimeIntentContext(
|
||||
text="海外市场资料太旧,重新检索 2026 年最新信息",
|
||||
requirement={"topic": task.title, "required_angles": task.required_angles, "revision": 1},
|
||||
plan_nodes=[{"id": "research", "role_key": "researcher"}],
|
||||
)
|
||||
)
|
||||
assert research.intent == "research_again"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scripted_quality_run_covers_ev01_annotations() -> None:
|
||||
task = get_eval_task("ev-01")
|
||||
store = MemoryReportCollaborationStore()
|
||||
session = await store.create_session(owner_id="u-eval", title=task.title, idempotency_key="eval-s")
|
||||
await store.save_requirement_state(
|
||||
session["id"],
|
||||
requirement_json={"snapshot": {"topic": task.title, "required_angles": task.required_angles, "time_range": task.time_range, "revision": 1}},
|
||||
requirement_revision=1,
|
||||
status="proposal_ready",
|
||||
)
|
||||
plan = {
|
||||
"id": "plan_eval_pipeline",
|
||||
"proposal_group_id": "grp_eval",
|
||||
"title": task.title,
|
||||
"strategy": "balanced_review",
|
||||
"summary": "eval",
|
||||
"rationale": "RC-BE-018",
|
||||
"recommended": True,
|
||||
"estimated_duration_seconds": 300,
|
||||
"estimated_cost_level": "low",
|
||||
"requirement_revision": 1,
|
||||
"roles": [],
|
||||
"nodes": [
|
||||
{"id": "research", "label": "检索", "role_key": "researcher", "output_artifact_type": "EvidenceBundle", "depends_on": [], "max_attempts": 3, "angle": task.required_angles[0]},
|
||||
{"id": "verify", "label": "核验", "role_key": "verifier", "output_artifact_type": "VerificationResult", "depends_on": ["research"], "max_attempts": 3},
|
||||
{"id": "analyze", "label": "分析", "role_key": "analyst", "output_artifact_type": "AngleAnalysis", "depends_on": ["verify"], "max_attempts": 3},
|
||||
{"id": "write", "label": "写作", "role_key": "writer", "output_artifact_type": "ReportSectionDraft", "depends_on": ["analyze"], "max_attempts": 3},
|
||||
{"id": "review", "label": "审稿", "role_key": "reviewer", "output_artifact_type": "ReviewDecision", "depends_on": ["write"], "max_attempts": 3},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "research", "target": "verify"},
|
||||
{"source": "verify", "target": "analyze"},
|
||||
{"source": "analyze", "target": "write"},
|
||||
{"source": "write", "target": "review"},
|
||||
],
|
||||
"quality_gates": [],
|
||||
"validation": {"ok": True, "errors": []},
|
||||
"revision": 1,
|
||||
"status": "proposed",
|
||||
}
|
||||
inserted = await store.insert_plan(session["id"], plan)
|
||||
await store.select_plan(session["id"], inserted["id"], idempotency_key="eval-sel", expected_revision=None)
|
||||
run = await store.create_run(session["id"], plan_id=inserted["id"], idempotency_key="eval-run", expected_revision=None)
|
||||
claims = []
|
||||
sources = []
|
||||
angle_blob = " ".join(task.required_angles)
|
||||
for fact in task.key_facts:
|
||||
token = fact.acceptable_source_tokens[0]
|
||||
claims.append(
|
||||
{
|
||||
"claim_id": fact.id,
|
||||
"text": f"{fact.text} {angle_blob}",
|
||||
"kind": "fact",
|
||||
"source_ids": [f"src-{fact.id}"],
|
||||
"excerpt": fact.text,
|
||||
"published_at": "2025-06-01",
|
||||
"unit": "万元" if "万" in fact.number else None,
|
||||
"scope": fact.id,
|
||||
}
|
||||
)
|
||||
sources.append({"source_id": f"src-{fact.id}", "title": f"{token} 公开口径 {angle_blob}", "excerpt": fact.text, "published_at": "2025-06-01"})
|
||||
kernel = QualityRoleKernel(store)
|
||||
kernel.enqueue(
|
||||
"research",
|
||||
TaskResult(
|
||||
kind="artifact_draft",
|
||||
artifact_type="EvidenceBundle",
|
||||
artifact_draft={"schema_version": 1, "claims": claims, "sources": sources, "conflicts": [], "coverage_gaps": []},
|
||||
),
|
||||
)
|
||||
status = await WaveExecutor(store, kernel).run_until_idle(run["id"])
|
||||
assert status == "completed"
|
||||
drafts = [item for item in await store.list_artifacts_full(run["id"]) if item["artifact_type"] == "ReportSectionDraft" and item["validation_status"] == "validated"]
|
||||
assert drafts
|
||||
markdown = str(drafts[-1]["content"]["markdown"])
|
||||
candidate = ReportCandidate(
|
||||
source="collaboration",
|
||||
markdown=markdown,
|
||||
source_index=[{"id": fact.id, "title": f"{fact.acceptable_source_tokens[0]} 公开口径", "url": "https://example.gov.cn/ev-01", "published_at": "2025-06-01", "added_at": "2025-06-02T00:00:00Z"} for fact in task.key_facts],
|
||||
events_present=["message.created", "report.version.created"],
|
||||
)
|
||||
score = score_report(task, candidate, directed_hits=[True, True, True], skip={"template_adherence"})
|
||||
assert score.metric("factual_accuracy").passed
|
||||
assert score.metric("angle_coverage").passed
|
||||
assert score.metric("citation_coverage").passed
|
||||
Reference in new issue
Block a user