feat(report-collaboration): complete RC-BE-015–018 backend, eval suite, and MySQL sync
Ship report rewrite/version APIs, security budget and audit persistence, AgentScope worker wiring, and the 20-task quality eval harness so the frontend can integrate against a stable contract once enabled in config. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
814a44f0b9
commit
5e953c9fcb
123 files changed
+26816
-3505
No files matched your search
@@ -0,0 +1,182 @@
|
||||
"""Frozen RC-BE-018 quality-eval contracts. No model calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
EvalCategory = Literal["policy", "market", "enterprise", "event", "comparison"]
|
||||
FailureKind = Literal["fabricated", "missing_angle", "uncited", "roundtable_essay", "over_budget", "empty", "illegal_json"]
|
||||
|
||||
|
||||
class KeyFact(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
text: str
|
||||
number: str
|
||||
must_cite: bool = True
|
||||
acceptable_source_tokens: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ForbiddenItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str
|
||||
reason: str
|
||||
|
||||
|
||||
class AcceptableSource(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title_tokens: list[str]
|
||||
domain_suffix: str | None = None
|
||||
min_year: int | None = 2024
|
||||
|
||||
|
||||
class TemplateSpec(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str
|
||||
sections: list[str]
|
||||
required_questions: list[str] = Field(default_factory=list)
|
||||
required_tables: list[str] = Field(default_factory=list)
|
||||
comparison_dimensions: list[str] = Field(default_factory=list)
|
||||
length_target: str | None = None
|
||||
|
||||
|
||||
class DirectedModification(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
command: str
|
||||
intent: str
|
||||
must_reopen_roles: list[str]
|
||||
must_not_reopen_roles: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class UsageSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
duration_seconds: float = 0
|
||||
model_calls: int = 0
|
||||
retrieval_calls: int = 0
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
cost: float = 0
|
||||
|
||||
|
||||
class ReportCandidate(BaseModel):
|
||||
"""A scoreable report, from the collaboration runtime or a roundtable export."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source: Literal["collaboration", "roundtable", "gold", "failure"] = "collaboration"
|
||||
markdown: str
|
||||
source_index: list[dict] = Field(default_factory=list)
|
||||
usage: UsageSnapshot | None = None
|
||||
events_present: list[str] = Field(default_factory=list)
|
||||
status: str = "completed"
|
||||
notes: str | None = None
|
||||
|
||||
|
||||
class EvalTask(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
category: EvalCategory
|
||||
title: str
|
||||
prompt: str
|
||||
time_range: str = "2024 年至今"
|
||||
audience: str = "企业战略决策人员"
|
||||
required_angles: list[str]
|
||||
key_facts: list[KeyFact]
|
||||
forbidden_fabrications: list[ForbiddenItem]
|
||||
template: TemplateSpec
|
||||
review_rules: list[str]
|
||||
acceptable_sources: list[AcceptableSource]
|
||||
directed_modifications: list[DirectedModification] = Field(default_factory=list)
|
||||
|
||||
|
||||
class MetricScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
score: float = Field(ge=0, le=1)
|
||||
passed: bool
|
||||
issues: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class TaskScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
task_id: str
|
||||
source: str
|
||||
metrics: list[MetricScore]
|
||||
passed: bool
|
||||
composite: float
|
||||
|
||||
def metric(self, name: str) -> MetricScore:
|
||||
for item in self.metrics:
|
||||
if item.name == name:
|
||||
return item
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
class ComparisonResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
task_id: str
|
||||
winner: Literal["collaboration", "roundtable", "tie"]
|
||||
collaboration: TaskScore
|
||||
roundtable: TaskScore
|
||||
deltas: dict[str, float]
|
||||
notes: str
|
||||
|
||||
|
||||
class ReliabilityCounts(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
runs: int
|
||||
early_stop: int = 0
|
||||
empty_output: int = 0
|
||||
illegal_json: int = 0
|
||||
missing_events: int = 0
|
||||
completed: int = 0
|
||||
|
||||
@property
|
||||
def failure_rate(self) -> float:
|
||||
if self.runs <= 0:
|
||||
return 1.0
|
||||
failed = self.early_stop + self.empty_output + self.illegal_json + self.missing_events
|
||||
return failed / self.runs
|
||||
|
||||
|
||||
class CatalogScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
tasks: int
|
||||
passed: int
|
||||
scores: list[TaskScore]
|
||||
composite: float
|
||||
by_category: dict[str, float]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AcceptableSource",
|
||||
"CatalogScore",
|
||||
"ComparisonResult",
|
||||
"DirectedModification",
|
||||
"EvalCategory",
|
||||
"EvalTask",
|
||||
"FailureKind",
|
||||
"ForbiddenItem",
|
||||
"KeyFact",
|
||||
"MetricScore",
|
||||
"ReliabilityCounts",
|
||||
"ReportCandidate",
|
||||
"TaskScore",
|
||||
"TemplateSpec",
|
||||
"UsageSnapshot",
|
||||
]
|
||||
Reference in new issue
Block a user