feat(report-collaboration): complete RC-BE-015–018 backend, eval suite, and MySQL sync
Ship report rewrite/version APIs, security budget and audit persistence, AgentScope worker wiring, and the 20-task quality eval harness so the frontend can integrate against a stable contract once enabled in config. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
814a44f0b9
commit
5e953c9fcb
123 files changed
+26816
-3505
No files matched your search
@@ -0,0 +1,29 @@
|
||||
"""Count early-stop / empty / illegal-JSON / missing-event outcomes across repeated runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.report_collaboration.eval.schema import ReliabilityCounts, ReportCandidate
|
||||
from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS
|
||||
|
||||
|
||||
def summarize_reliability(runs: list[ReportCandidate]) -> ReliabilityCounts:
|
||||
counts = ReliabilityCounts(runs=len(runs))
|
||||
for item in runs:
|
||||
empty = not (item.markdown or "").strip()
|
||||
illegal = item.notes == "illegal_json" or ((item.markdown or "").lstrip().startswith("{") and "schema_version" not in (item.markdown or ""))
|
||||
missing = bool(item.events_present) and any(name not in set(item.events_present) for name in REQUIRED_EVENTS)
|
||||
early = item.status not in {"completed", "cancelled"} or item.notes == "early_stop"
|
||||
if empty:
|
||||
counts.empty_output += 1
|
||||
elif illegal:
|
||||
counts.illegal_json += 1
|
||||
elif missing:
|
||||
counts.missing_events += 1
|
||||
elif early:
|
||||
counts.early_stop += 1
|
||||
elif item.status == "completed":
|
||||
counts.completed += 1
|
||||
return counts
|
||||
|
||||
|
||||
__all__ = ["summarize_reliability"]
|
||||
Reference in new issue
Block a user