feat(report-collaboration): complete RC-BE-015–018 backend, eval suite, and MySQL sync
Ship report rewrite/version APIs, security budget and audit persistence, AgentScope worker wiring, and the 20-task quality eval harness so the frontend can integrate against a stable contract once enabled in config. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
814a44f0b9
commit
5e953c9fcb
123 files changed
+26816
-3505
No files matched your search
@@ -0,0 +1,22 @@
|
||||
"""RC-BE-018 quality eval: annotated tasks, deterministic scoring, blind compare."""
|
||||
|
||||
from app.report_collaboration.eval.catalog import get_eval_task, load_eval_tasks
|
||||
from app.report_collaboration.eval.compare import compare_blind
|
||||
from app.report_collaboration.eval.materialize import materialize_failure, materialize_gold
|
||||
from app.report_collaboration.eval.reliability import summarize_reliability
|
||||
from app.report_collaboration.eval.scorer import evaluate_catalog, evaluate_task, score_report
|
||||
from app.report_collaboration.eval.thresholds import METRIC_GATES, RECOMMENDED_RUNTIME_PROFILE
|
||||
|
||||
__all__ = [
|
||||
"METRIC_GATES",
|
||||
"RECOMMENDED_RUNTIME_PROFILE",
|
||||
"compare_blind",
|
||||
"evaluate_catalog",
|
||||
"evaluate_task",
|
||||
"get_eval_task",
|
||||
"load_eval_tasks",
|
||||
"materialize_failure",
|
||||
"materialize_gold",
|
||||
"score_report",
|
||||
"summarize_reliability",
|
||||
]
|
||||
@@ -0,0 +1,416 @@
|
||||
"""Twenty annotated quality-eval tasks. Categories are balanced 4×5."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.report_collaboration.eval.schema import (
|
||||
AcceptableSource,
|
||||
DirectedModification,
|
||||
EvalTask,
|
||||
ForbiddenItem,
|
||||
KeyFact,
|
||||
TemplateSpec,
|
||||
)
|
||||
|
||||
STANDARD_DIRECTED_MODS = [
|
||||
DirectedModification(
|
||||
id="add-supply-chain",
|
||||
command="增加供应链与原材料风险角度",
|
||||
intent="add_angle",
|
||||
must_reopen_roles=["researcher"],
|
||||
must_not_reopen_roles=[],
|
||||
),
|
||||
DirectedModification(
|
||||
id="re-search-2026",
|
||||
command="海外市场资料太旧,重新检索 2026 年最新信息",
|
||||
intent="research_again",
|
||||
must_reopen_roles=["researcher"],
|
||||
must_not_reopen_roles=[],
|
||||
),
|
||||
DirectedModification(
|
||||
id="rewrite-ch2",
|
||||
command="重写第二章,结论更谨慎,不要重新检索",
|
||||
intent="rewrite_section",
|
||||
must_reopen_roles=["writer"],
|
||||
must_not_reopen_roles=["researcher"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _fact(fact_id: str, text: str, number: str, *tokens: str) -> KeyFact:
|
||||
return KeyFact(id=fact_id, text=text, number=number, acceptable_source_tokens=list(tokens))
|
||||
|
||||
|
||||
def _src(*tokens: str, year: int = 2024, domain: str | None = None) -> AcceptableSource:
|
||||
return AcceptableSource(title_tokens=list(tokens), domain_suffix=domain, min_year=year)
|
||||
|
||||
|
||||
def _task(
|
||||
task_id: str,
|
||||
category: str,
|
||||
title: str,
|
||||
prompt: str,
|
||||
*,
|
||||
angles: list[str],
|
||||
facts: list[KeyFact],
|
||||
forbidden: list[tuple[str, str]],
|
||||
sections: list[str],
|
||||
questions: list[str],
|
||||
sources: list[AcceptableSource],
|
||||
tables: list[str] | None = None,
|
||||
dimensions: list[str] | None = None,
|
||||
time_range: str = "2024 年至今",
|
||||
review: list[str] | None = None,
|
||||
) -> EvalTask:
|
||||
return EvalTask(
|
||||
id=task_id,
|
||||
category=category, # type: ignore[arg-type]
|
||||
title=title,
|
||||
prompt=prompt,
|
||||
time_range=time_range,
|
||||
required_angles=angles,
|
||||
key_facts=facts,
|
||||
forbidden_fabrications=[ForbiddenItem(text=text, reason=reason) for text, reason in forbidden],
|
||||
template=TemplateSpec(
|
||||
title=title,
|
||||
sections=sections,
|
||||
required_questions=questions,
|
||||
required_tables=tables or [],
|
||||
comparison_dimensions=dimensions or [],
|
||||
),
|
||||
review_rules=review
|
||||
or [
|
||||
"关键数字必须带来源 ID",
|
||||
"禁止采用来源中不存在的数字或事件归属",
|
||||
"审稿问题必须映射到具体章节或写作节点",
|
||||
],
|
||||
acceptable_sources=sources,
|
||||
directed_modifications=list(STANDARD_DIRECTED_MODS),
|
||||
)
|
||||
|
||||
|
||||
def load_eval_tasks() -> list[EvalTask]:
|
||||
return [
|
||||
_task(
|
||||
"ev-01",
|
||||
"policy",
|
||||
"新能源车购置税延续政策影响",
|
||||
"评估 2024 年至今新能源汽车购置税减免延续对企业定价、销量与地方补贴叠加的影响,面向战略决策人员,关键数字必须给出来源。",
|
||||
angles=["购置税政策", "地方补贴", "销量传导"],
|
||||
facts=[
|
||||
_fact("f-tax-rate", "购置税减免按 3 万元限额执行", "3 万", "财政部", "税务总局"),
|
||||
_fact("f-tax-end", "现行减免安排覆盖至 2027 年底", "2027", "财政部"),
|
||||
],
|
||||
forbidden=[("购置税永久取消", "政策文件未宣布永久取消")],
|
||||
sections=["政策要点", "销量与定价传导", "结论与局限"],
|
||||
questions=["减免限额是多少?"],
|
||||
sources=[_src("财政部", "税务总局", year=2024, domain="gov.cn")],
|
||||
),
|
||||
_task(
|
||||
"ev-02",
|
||||
"policy",
|
||||
"数据跨境新规对车企出海影响",
|
||||
"分析 2024 年以来数据出境安全评估与标准合同对车企海外座舱数据回流的影响。",
|
||||
angles=["出境评估", "座舱数据", "合规成本"],
|
||||
facts=[
|
||||
_fact("f-cac", "重要数据出境需通过安全评估", "安全评估", "网信办"),
|
||||
_fact("f-scc", "标准合同备案是中小流量路径", "标准合同", "网信办"),
|
||||
],
|
||||
forbidden=[("所有车企已被禁止数据出境", "无此禁令")],
|
||||
sections=["监管框架", "出海运营影响", "结论与局限"],
|
||||
questions=["中小流量可用哪条路径?"],
|
||||
sources=[_src("网信办", year=2024, domain="gov.cn")],
|
||||
),
|
||||
_task(
|
||||
"ev-03",
|
||||
"policy",
|
||||
"地方以旧换新补贴差异",
|
||||
"比较 2024–2026 年主要省市汽车以旧换新补贴力度与兑付节奏。",
|
||||
angles=["补贴力度", "兑付节奏", "地方财政"],
|
||||
facts=[
|
||||
_fact("f-bj", "北京最高补贴约 1.5 万元", "1.5 万", "北京", "商务委"),
|
||||
_fact("f-gd", "广东叠加国补后上限约 2 万元", "2 万", "广东", "商务厅"),
|
||||
],
|
||||
forbidden=[("全国统一补贴 5 万元", "无全国统一 5 万标准")],
|
||||
sections=["补贴对照", "财政可持续性", "结论与局限"],
|
||||
questions=["两地最高补贴分别是多少?"],
|
||||
sources=[_src("商务委"), _src("商务厅")],
|
||||
tables=["补贴对照"],
|
||||
dimensions=["北京", "广东"],
|
||||
),
|
||||
_task(
|
||||
"ev-04",
|
||||
"policy",
|
||||
"欧盟 CBAM 对动力电池出口",
|
||||
"评估欧盟碳边境调节机制对动力电池出口报价与认证周期的影响。",
|
||||
angles=["碳足迹", "出口报价", "认证周期"],
|
||||
facts=[
|
||||
_fact("f-cbam", "过渡期报告义务持续至 2025 年底", "2025", "欧盟委员会", "CBAM"),
|
||||
_fact("f-cbam-pay", "正式缴税阶段自 2026 年起", "2026", "欧盟委员会"),
|
||||
],
|
||||
forbidden=[("中国电池已被欧盟全面禁止进口", "无此禁令")],
|
||||
sections=["规则时间表", "出口成本", "结论与局限"],
|
||||
questions=["正式缴税从哪一年开始?"],
|
||||
sources=[_src("欧盟委员会", "CBAM", year=2024, domain="europa.eu")],
|
||||
),
|
||||
_task(
|
||||
"ev-05",
|
||||
"market",
|
||||
"新能源汽车主要品牌市场趋势",
|
||||
"撰写一份新能源汽车主要品牌市场趋势分析报告,覆盖 2024 年至今的市场份额、价格策略、技术路线、海外进展、政策环境和未来 12~24 个月风险与机会,面向企业战略决策人员,要求所有关键数字给出来源。",
|
||||
angles=["市场份额", "价格策略", "技术路线", "海外进展", "政策环境", "风险与机会"],
|
||||
facts=[
|
||||
_fact("f-share", "2025 年新能源乘用车零售渗透率约 51%", "51%", "乘联会"),
|
||||
_fact("f-price", "主流纯电成交价带下移至 12 万元附近", "12 万", "乘联会"),
|
||||
],
|
||||
forbidden=[("某品牌全球份额已达 80%", "无公开口径支持")],
|
||||
sections=["市场格局", "价格与技术", "海外与政策", "风险与机会", "结论与局限"],
|
||||
questions=["零售渗透率是多少?"],
|
||||
sources=[_src("乘联会", year=2025)],
|
||||
tables=["品牌份额"],
|
||||
),
|
||||
_task(
|
||||
"ev-06",
|
||||
"market",
|
||||
"固态电池产业化节奏",
|
||||
"梳理 2024 至今半固态/全固态电池量产节点与车型搭载计划。",
|
||||
angles=["量产节点", "车型搭载", "成本曲线"],
|
||||
facts=[
|
||||
_fact("f-semi", "半固态已有小批量车型搭载", "小批量", "工信部", "车企年报"),
|
||||
_fact("f-all-solid", "全固态大规模量产仍多指向 2027 年后", "2027", "工信部"),
|
||||
],
|
||||
forbidden=[("全固态已在 2024 年全面取代液态锂电", "与公开量产节奏不符")],
|
||||
sections=["技术成熟度", "车型计划", "结论与局限"],
|
||||
questions=["全固态大规模量产多指向哪一年之后?"],
|
||||
sources=[_src("工信部"), _src("年报")],
|
||||
),
|
||||
_task(
|
||||
"ev-07",
|
||||
"market",
|
||||
"新型储能装机与电价",
|
||||
"分析 2024 至今新型储能新增装机与峰谷价差对独立储能收益的影响。",
|
||||
angles=["新增装机", "峰谷价差", "独立储能收益"],
|
||||
facts=[
|
||||
_fact("f-storage", "2024 年新型储能新增装机约 42 GW", "42", "国家能源局"),
|
||||
_fact("f-spread", "部分省份峰谷价差仍高于 0.7 元/千瓦时", "0.7", "电网", "能源局"),
|
||||
],
|
||||
forbidden=[("全国统一现货电价已取消峰谷", "与现行分时电价不符")],
|
||||
sections=["装机与市场", "收益敏感性", "结论与局限"],
|
||||
questions=["2024 年新增装机大约多少?"],
|
||||
sources=[_src("国家能源局", year=2024, domain="gov.cn")],
|
||||
),
|
||||
_task(
|
||||
"ev-08",
|
||||
"market",
|
||||
"跨境电商物流成本",
|
||||
"评估 2024 至今跨境电商头程海运与末端派送成本变化。",
|
||||
angles=["头程海运", "末端派送", "时效"],
|
||||
facts=[
|
||||
_fact("f-ocean", "亚洲至美西 40 尺柜现货运价一度回落至 2000 美元附近", "2000", "上海航运交易所"),
|
||||
_fact("f-last", "欧美末端派送单票仍多在 6 美元以上", "6", "物流年报"),
|
||||
],
|
||||
forbidden=[("全球海运已免费", "无此事实")],
|
||||
sections=["成本结构", "时效风险", "结论与局限"],
|
||||
questions=["美西现货柜价大约落在什么水平?"],
|
||||
sources=[_src("上海航运交易所"), _src("物流年报")],
|
||||
),
|
||||
_task(
|
||||
"ev-09",
|
||||
"enterprise",
|
||||
"车企海外工厂盈利能力",
|
||||
"研究某头部车企 2024 至今海外工厂产能利用率与单车利润。",
|
||||
angles=["产能利用率", "单车利润", "本地化采购"],
|
||||
facts=[
|
||||
_fact("f-util", "欧洲工厂利用率公开口径约 60%", "60%", "车企年报"),
|
||||
_fact("f-margin", "海外单车利润仍低于国内约 8000 元", "8000", "车企年报"),
|
||||
],
|
||||
forbidden=[("海外工厂已实现 40% 净利率", "远超公开年报")],
|
||||
sections=["产能与利用率", "利润拆解", "结论与局限"],
|
||||
questions=["欧洲工厂利用率公开口径是多少?"],
|
||||
sources=[_src("年报", "车企")],
|
||||
),
|
||||
_task(
|
||||
"ev-10",
|
||||
"enterprise",
|
||||
"平台广告收入结构",
|
||||
"拆解某互联网平台 2025 年广告收入中搜索、信息流与电商广告占比。",
|
||||
angles=["搜索广告", "信息流广告", "电商广告"],
|
||||
facts=[
|
||||
_fact("f-ad-mix", "信息流广告约占广告收入 55%", "55%", "平台财报"),
|
||||
_fact("f-ad-growth", "电商广告同比增速约 18%", "18%", "平台财报"),
|
||||
],
|
||||
forbidden=[("广告收入已超过 1 万亿美元", "数量级不可信")],
|
||||
sections=["收入结构", "增速与风险", "结论与局限"],
|
||||
questions=["信息流广告占比大约多少?"],
|
||||
sources=[_src("财报", "平台")],
|
||||
),
|
||||
_task(
|
||||
"ev-11",
|
||||
"enterprise",
|
||||
"运营商算力资本开支",
|
||||
"评估三大运营商 2024–2026 年智算资本开支指引与机柜上架率。",
|
||||
angles=["资本开支", "上架率", "智算供给"],
|
||||
facts=[
|
||||
_fact("f-capex", "三家合计智算相关开支公开指引约 450 亿元", "450", "运营商年报"),
|
||||
_fact("f-rack", "东部枢纽上架率公开口径约 70%", "70%", "运营商年报"),
|
||||
],
|
||||
forbidden=[("三家已宣布停止全部 5G 投资", "与年报不符")],
|
||||
sections=["开支指引", "供给转化", "结论与局限"],
|
||||
questions=["智算相关开支公开指引大约多少?"],
|
||||
sources=[_src("运营商", "年报")],
|
||||
),
|
||||
_task(
|
||||
"ev-12",
|
||||
"enterprise",
|
||||
"创新药管线估值敏感点",
|
||||
"分析某药企核心管线峰值销售假设与专利悬崖时间。",
|
||||
angles=["峰值销售", "专利悬崖", "适应症扩张"],
|
||||
facts=[
|
||||
_fact("f-peak", "核心品种峰值销售卖方一致预期约 80 亿元", "80 亿", "卖方研报"),
|
||||
_fact("f-patent", "核心化合物专利到期窗口在 2031 年", "2031", "专利", "年报"),
|
||||
],
|
||||
forbidden=[("该品种已获全球所有适应症批准", "过度概括")],
|
||||
sections=["估值假设", "专利与竞争", "结论与局限"],
|
||||
questions=["专利到期窗口在哪一年?"],
|
||||
sources=[_src("年报"), _src("研报")],
|
||||
),
|
||||
_task(
|
||||
"ev-13",
|
||||
"event",
|
||||
"港口拥堵对汽车供应链",
|
||||
"研判 2026 年某枢纽港拥堵对整车出口周期与零部件库存的冲击。",
|
||||
angles=["出口周期", "零部件库存", "替代港口"],
|
||||
facts=[
|
||||
_fact("f-wait", "平均等泊时间升至 7 天", "7 天", "港口公告"),
|
||||
_fact("f-inv", "关键电子件安全库存被抬至 28 天", "28 天", "车企公告"),
|
||||
],
|
||||
forbidden=[("全球汽车出口已中断三个月", "无此范围事实")],
|
||||
sections=["事件经过", "供应链冲击", "结论与局限"],
|
||||
questions=["平均等泊时间升至几天?"],
|
||||
sources=[_src("港口公告", year=2026), _src("车企公告", year=2026)],
|
||||
time_range="2026 年",
|
||||
),
|
||||
_task(
|
||||
"ev-14",
|
||||
"event",
|
||||
"重大安全事件复盘",
|
||||
"复盘某智能驾驶相关安全事件的责任划分、召回规模与监管动作。",
|
||||
angles=["责任划分", "召回规模", "监管动作"],
|
||||
facts=[
|
||||
_fact("f-recall", "召回涉及约 12 万辆", "12 万", "市监总局"),
|
||||
_fact("f-reg", "监管要求限期提交软件整改报告", "整改报告", "市监总局"),
|
||||
],
|
||||
forbidden=[("已认定驾驶员无任何责任", "调查结论未如此表述")],
|
||||
sections=["事件认定", "召回与整改", "结论与局限"],
|
||||
questions=["召回大约涉及多少辆?"],
|
||||
sources=[_src("市监总局", year=2025, domain="gov.cn")],
|
||||
),
|
||||
_task(
|
||||
"ev-15",
|
||||
"event",
|
||||
"汇率波动对出口结算",
|
||||
"评估人民币对美元一轮 3% 波动对车企出口结算与套期保值的影响。",
|
||||
angles=["结算货币", "套保覆盖", "利润弹性"],
|
||||
facts=[
|
||||
_fact("f-fx", "观察期内人民币对美元波动约 3%", "3%", "外汇交易中心"),
|
||||
_fact("f-hedge", "样本车企套保覆盖率约 40%", "40%", "车企年报"),
|
||||
],
|
||||
forbidden=[("车企出口全部以人民币结算", "与公开披露不符")],
|
||||
sections=["波动事实", "财务影响", "结论与局限"],
|
||||
questions=["套保覆盖率大约多少?"],
|
||||
sources=[_src("外汇交易中心"), _src("年报")],
|
||||
),
|
||||
_task(
|
||||
"ev-16",
|
||||
"event",
|
||||
"跨境并购审批进展",
|
||||
"跟踪某电池材料跨境收购的反垄断与外资安全审查节点。",
|
||||
angles=["反垄断", "外资安全审查", "交割条件"],
|
||||
facts=[
|
||||
_fact("f-am", "申报方披露审查已进入第二阶段", "第二阶段", "公司公告"),
|
||||
_fact("f-close", "交割先决条件包括当地政府无异议函", "无异议函", "公司公告"),
|
||||
],
|
||||
forbidden=[("交易已无条件获批并完成交割", "与进行中审查矛盾")],
|
||||
sections=["审查节点", "交割风险", "结论与局限"],
|
||||
questions=["审查目前进入第几阶段?"],
|
||||
sources=[_src("公司公告", year=2025)],
|
||||
),
|
||||
_task(
|
||||
"ev-17",
|
||||
"comparison",
|
||||
"三家车企海外进展对比",
|
||||
"对比三家主流车企 2024 至今海外销量、工厂与渠道密度。",
|
||||
angles=["海外销量", "工厂布局", "渠道密度"],
|
||||
facts=[
|
||||
_fact("f-vol-a", "A 企海外销量约 40 万辆", "40 万", "A 企年报"),
|
||||
_fact("f-vol-b", "B 企海外销量约 28 万辆", "28 万", "B 企年报"),
|
||||
_fact("f-vol-c", "C 企海外销量约 15 万辆", "15 万", "C 企年报"),
|
||||
],
|
||||
forbidden=[("三家海外销量完全相同", "抹平差异")],
|
||||
sections=["对比总览", "分项解读", "结论与局限"],
|
||||
questions=["三家海外销量分别是多少?"],
|
||||
sources=[_src("A 企年报"), _src("B 企年报"), _src("C 企年报")],
|
||||
tables=["海外对照"],
|
||||
dimensions=["销量", "工厂", "渠道"],
|
||||
),
|
||||
_task(
|
||||
"ev-18",
|
||||
"comparison",
|
||||
"磷酸铁锂与三元成本对比",
|
||||
"对比磷酸铁锂与三元路线 2025 年电芯成本、能量密度与出口认证。",
|
||||
angles=["电芯成本", "能量密度", "出口认证"],
|
||||
facts=[
|
||||
_fact("f-lfp", "磷酸铁锂电芯成本约 0.38 元/Wh", "0.38", "行业月报"),
|
||||
_fact("f-ncm", "三元电芯成本约 0.52 元/Wh", "0.52", "行业月报"),
|
||||
],
|
||||
forbidden=[("三元成本已低于磷酸铁锂", "与引用口径冲突")],
|
||||
sections=["成本对照", "性能与认证", "结论与局限"],
|
||||
questions=["两条路线电芯成本分别是多少?"],
|
||||
sources=[_src("行业月报", year=2025)],
|
||||
tables=["成本对照"],
|
||||
dimensions=["成本", "密度"],
|
||||
),
|
||||
_task(
|
||||
"ev-19",
|
||||
"comparison",
|
||||
"两地充电补贴政策对比",
|
||||
"对比两座城市公共充电桩建设补贴与服务费上限。",
|
||||
angles=["建设补贴", "服务费上限", "利用率"],
|
||||
facts=[
|
||||
_fact("f-sub-a", "甲市桩补最高 400 元/千瓦", "400", "发改委"),
|
||||
_fact("f-sub-b", "乙市服务费上限 0.8 元/千瓦时", "0.8", "发改委"),
|
||||
],
|
||||
forbidden=[("两地已取消全部充电补贴", "与现行文件不符")],
|
||||
sections=["政策对照", "运营含义", "结论与局限"],
|
||||
questions=["甲市桩补上限是多少?"],
|
||||
sources=[_src("发改委", year=2025, domain="gov.cn")],
|
||||
tables=["补贴对照"],
|
||||
dimensions=["甲市", "乙市"],
|
||||
),
|
||||
_task(
|
||||
"ev-20",
|
||||
"comparison",
|
||||
"三家云厂商 GPU 供给对比",
|
||||
"对比三家云厂商 2025 年可售智算卡供给、单价与交付周期。",
|
||||
angles=["可售供给", "单价", "交付周期"],
|
||||
facts=[
|
||||
_fact("f-gpu-a", "甲云公开可售高端卡约 2 万张", "2 万", "云厂商财报"),
|
||||
_fact("f-gpu-b", "乙云交付周期约 10 周", "10 周", "云厂商财报"),
|
||||
],
|
||||
forbidden=[("三家均可立刻交付百万张同等高端卡", "供给夸张")],
|
||||
sections=["供给对照", "价格与交期", "结论与局限"],
|
||||
questions=["乙云交付周期大约多久?"],
|
||||
sources=[_src("云厂商", "财报", year=2025)],
|
||||
tables=["供给对照"],
|
||||
dimensions=["供给", "交期"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def get_eval_task(task_id: str) -> EvalTask:
|
||||
for task in load_eval_tasks():
|
||||
if task.id == task_id:
|
||||
return task
|
||||
raise KeyError(task_id)
|
||||
|
||||
|
||||
__all__ = ["STANDARD_DIRECTED_MODS", "get_eval_task", "load_eval_tasks"]
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Blind comparison between a collaboration report and a roundtable export."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.report_collaboration.eval.schema import ComparisonResult, EvalTask, ReportCandidate, TaskScore
|
||||
from app.report_collaboration.eval.scorer import score_report
|
||||
|
||||
|
||||
def compare_blind(
|
||||
task: EvalTask,
|
||||
collaboration: ReportCandidate,
|
||||
roundtable: ReportCandidate,
|
||||
*,
|
||||
directed_hits: list[bool] | None = None,
|
||||
) -> ComparisonResult:
|
||||
left = score_report(task, collaboration.model_copy(update={"source": "collaboration"}), directed_hits=directed_hits)
|
||||
right = score_report(task, roundtable.model_copy(update={"source": "roundtable"}), directed_hits=directed_hits)
|
||||
deltas = {item.name: round(item.score - right.metric(item.name).score, 4) for item in left.metrics}
|
||||
if left.composite > right.composite + 1e-9:
|
||||
winner = "collaboration"
|
||||
elif right.composite > left.composite + 1e-9:
|
||||
winner = "roundtable"
|
||||
else:
|
||||
winner = "tie"
|
||||
notes = _notes(winner, left, right, deltas)
|
||||
return ComparisonResult(task_id=task.id, winner=winner, collaboration=left, roundtable=right, deltas=deltas, notes=notes)
|
||||
|
||||
|
||||
def _notes(winner: str, left: TaskScore, right: TaskScore, deltas: dict[str, float]) -> str:
|
||||
drivers = [name for name, delta in sorted(deltas.items(), key=lambda item: item[1], reverse=True) if delta > 0][:3]
|
||||
if winner == "collaboration":
|
||||
return "协作报告在 " + "、".join(drivers or ["综合分"]) + " 上优于会商导出;上线不得仅以「已接入 AgentScope」为由。"
|
||||
if winner == "roundtable":
|
||||
return "会商导出综合分更高。按决策门,本任务不能作为协作方案上线依据。"
|
||||
return f"综合分持平({left.composite} vs {right.composite})。"
|
||||
|
||||
|
||||
__all__ = ["compare_blind"]
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Build gold reports and reproducible failure instances from task annotations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.report_collaboration.eval.schema import EvalTask, FailureKind, ReportCandidate, UsageSnapshot
|
||||
from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS
|
||||
|
||||
|
||||
def materialize_gold(task: EvalTask) -> ReportCandidate:
|
||||
sections = list(task.template.sections)
|
||||
body: list[str] = [f"# {sections[0]}"]
|
||||
body.append(f"{task.title}。时间范围:{task.time_range}。受众:{task.audience}。")
|
||||
body.append("必选角度:" + "、".join(task.required_angles) + "。")
|
||||
for fact in task.key_facts:
|
||||
body.append(f"{fact.text}({fact.id})。")
|
||||
if task.template.required_questions:
|
||||
body.append("必答问题:" + " ".join(task.template.required_questions))
|
||||
body.append("回答:" + ";".join(f"{fact.number}" for fact in task.key_facts[:2]))
|
||||
if task.template.required_tables or task.template.comparison_dimensions:
|
||||
headers = task.template.comparison_dimensions or ["项目", "口径"]
|
||||
if len(headers) == 1:
|
||||
headers = [headers[0], "取值"]
|
||||
body.append("| " + " | ".join(headers) + " |")
|
||||
body.append("| " + " | ".join("---" for _ in headers) + " |")
|
||||
row = [task.key_facts[0].number if task.key_facts else "—"]
|
||||
while len(row) < len(headers):
|
||||
row.append(task.required_angles[0] if task.required_angles else "—")
|
||||
body.append("| " + " | ".join(row[: len(headers)]) + " |")
|
||||
for title in sections[1:]:
|
||||
body.append(f"# {title}")
|
||||
body.append(f"{title}需结合已引用主张审慎判断,避免把未核验数字写成事实。")
|
||||
if title == sections[-1]:
|
||||
body.append("局限:公开口径可能修订;本报告不采用来源中不存在的数字或事件归属。")
|
||||
source_index = []
|
||||
for index, source in enumerate(task.acceptable_sources, start=1):
|
||||
token = source.title_tokens[0] if source.title_tokens else f"source-{index}"
|
||||
year = source.min_year or 2025
|
||||
source_index.append(
|
||||
{
|
||||
"id": task.key_facts[min(index - 1, len(task.key_facts) - 1)].id if task.key_facts else f"src-{index}",
|
||||
"title": f"{token} 公开口径",
|
||||
"url": f"https://example.{source.domain_suffix or 'org'}/{task.id}/{index}",
|
||||
"published_at": f"{year}-06-01",
|
||||
"added_at": f"{year}-06-02T00:00:00Z",
|
||||
"agent_run_id": None,
|
||||
}
|
||||
)
|
||||
return ReportCandidate(
|
||||
source="gold",
|
||||
markdown="\n\n".join(body),
|
||||
source_index=source_index,
|
||||
events_present=list(REQUIRED_EVENTS),
|
||||
status="completed",
|
||||
notes="annotation-built gold",
|
||||
)
|
||||
|
||||
|
||||
def materialize_failure(task: EvalTask, kind: FailureKind) -> ReportCandidate:
|
||||
gold = materialize_gold(task)
|
||||
if kind == "fabricated" and task.forbidden_fabrications:
|
||||
return gold.model_copy(update={"source": "failure", "markdown": gold.markdown + "\n\n" + task.forbidden_fabrications[0].text, "notes": kind})
|
||||
if kind == "missing_angle" and task.required_angles:
|
||||
return gold.model_copy(update={"source": "failure", "markdown": gold.markdown.replace(task.required_angles[0], ""), "notes": kind})
|
||||
if kind == "uncited":
|
||||
markdown = gold.markdown
|
||||
for fact in task.key_facts:
|
||||
markdown = markdown.replace(f"({fact.id})", "")
|
||||
return gold.model_copy(update={"source": "failure", "markdown": markdown, "source_index": [], "notes": kind})
|
||||
if kind == "roundtable_essay":
|
||||
snippet = task.key_facts[0].text if task.key_facts else task.title
|
||||
return ReportCandidate(
|
||||
source="roundtable",
|
||||
markdown=f"{task.title}总体向好。{snippet}。建议继续观察。",
|
||||
source_index=[],
|
||||
events_present=[],
|
||||
status="completed",
|
||||
notes="roundtable-style essay without citations or template",
|
||||
)
|
||||
if kind == "over_budget":
|
||||
return gold.model_copy(
|
||||
update={
|
||||
"source": "failure",
|
||||
"usage": UsageSnapshot(duration_seconds=9000, model_calls=400, retrieval_calls=80, input_tokens=300_000, output_tokens=80_000, cost=12.5),
|
||||
"notes": kind,
|
||||
}
|
||||
)
|
||||
if kind == "empty":
|
||||
return ReportCandidate(source="failure", markdown="", source_index=[], status="failed", notes=kind)
|
||||
if kind == "illegal_json":
|
||||
return ReportCandidate(source="failure", markdown="{not-json", source_index=[], status="failed", events_present=[], notes=kind)
|
||||
return gold.model_copy(update={"source": "failure", "markdown": "", "notes": kind})
|
||||
|
||||
|
||||
__all__ = ["materialize_failure", "materialize_gold"]
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Canonical quality-chain plan used to score directed modifications."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def eval_quality_plan() -> dict:
|
||||
nodes = [
|
||||
{
|
||||
"id": "research",
|
||||
"label": "检索",
|
||||
"role_key": "researcher",
|
||||
"angle": "海外进展",
|
||||
"output_artifact_type": "EvidenceBundle",
|
||||
"depends_on": [],
|
||||
},
|
||||
{
|
||||
"id": "verify",
|
||||
"label": "核验",
|
||||
"role_key": "verifier",
|
||||
"output_artifact_type": "VerificationResult",
|
||||
"depends_on": ["research"],
|
||||
},
|
||||
{
|
||||
"id": "analyze",
|
||||
"label": "分析",
|
||||
"role_key": "analyst",
|
||||
"output_artifact_type": "AngleAnalysis",
|
||||
"depends_on": ["verify"],
|
||||
},
|
||||
{
|
||||
"id": "write",
|
||||
"label": "写作",
|
||||
"role_key": "writer",
|
||||
"output_artifact_type": "ReportSectionDraft",
|
||||
"depends_on": ["analyze"],
|
||||
},
|
||||
{
|
||||
"id": "review",
|
||||
"label": "审稿",
|
||||
"role_key": "reviewer",
|
||||
"output_artifact_type": "ReviewDecision",
|
||||
"depends_on": ["write"],
|
||||
},
|
||||
]
|
||||
return {
|
||||
"id": "plan_eval",
|
||||
"nodes": nodes,
|
||||
"edges": [
|
||||
{"source": "research", "target": "verify"},
|
||||
{"source": "verify", "target": "analyze"},
|
||||
{"source": "analyze", "target": "write"},
|
||||
{"source": "write", "target": "review"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["eval_quality_plan"]
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Count early-stop / empty / illegal-JSON / missing-event outcomes across repeated runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.report_collaboration.eval.schema import ReliabilityCounts, ReportCandidate
|
||||
from app.report_collaboration.eval.thresholds import REQUIRED_EVENTS
|
||||
|
||||
|
||||
def summarize_reliability(runs: list[ReportCandidate]) -> ReliabilityCounts:
|
||||
counts = ReliabilityCounts(runs=len(runs))
|
||||
for item in runs:
|
||||
empty = not (item.markdown or "").strip()
|
||||
illegal = item.notes == "illegal_json" or ((item.markdown or "").lstrip().startswith("{") and "schema_version" not in (item.markdown or ""))
|
||||
missing = bool(item.events_present) and any(name not in set(item.events_present) for name in REQUIRED_EVENTS)
|
||||
early = item.status not in {"completed", "cancelled"} or item.notes == "early_stop"
|
||||
if empty:
|
||||
counts.empty_output += 1
|
||||
elif illegal:
|
||||
counts.illegal_json += 1
|
||||
elif missing:
|
||||
counts.missing_events += 1
|
||||
elif early:
|
||||
counts.early_stop += 1
|
||||
elif item.status == "completed":
|
||||
counts.completed += 1
|
||||
return counts
|
||||
|
||||
|
||||
__all__ = ["summarize_reliability"]
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Frozen RC-BE-018 quality-eval contracts. No model calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
EvalCategory = Literal["policy", "market", "enterprise", "event", "comparison"]
|
||||
FailureKind = Literal["fabricated", "missing_angle", "uncited", "roundtable_essay", "over_budget", "empty", "illegal_json"]
|
||||
|
||||
|
||||
class KeyFact(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
text: str
|
||||
number: str
|
||||
must_cite: bool = True
|
||||
acceptable_source_tokens: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ForbiddenItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str
|
||||
reason: str
|
||||
|
||||
|
||||
class AcceptableSource(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title_tokens: list[str]
|
||||
domain_suffix: str | None = None
|
||||
min_year: int | None = 2024
|
||||
|
||||
|
||||
class TemplateSpec(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str
|
||||
sections: list[str]
|
||||
required_questions: list[str] = Field(default_factory=list)
|
||||
required_tables: list[str] = Field(default_factory=list)
|
||||
comparison_dimensions: list[str] = Field(default_factory=list)
|
||||
length_target: str | None = None
|
||||
|
||||
|
||||
class DirectedModification(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
command: str
|
||||
intent: str
|
||||
must_reopen_roles: list[str]
|
||||
must_not_reopen_roles: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class UsageSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
duration_seconds: float = 0
|
||||
model_calls: int = 0
|
||||
retrieval_calls: int = 0
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
cost: float = 0
|
||||
|
||||
|
||||
class ReportCandidate(BaseModel):
|
||||
"""A scoreable report, from the collaboration runtime or a roundtable export."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source: Literal["collaboration", "roundtable", "gold", "failure"] = "collaboration"
|
||||
markdown: str
|
||||
source_index: list[dict] = Field(default_factory=list)
|
||||
usage: UsageSnapshot | None = None
|
||||
events_present: list[str] = Field(default_factory=list)
|
||||
status: str = "completed"
|
||||
notes: str | None = None
|
||||
|
||||
|
||||
class EvalTask(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
category: EvalCategory
|
||||
title: str
|
||||
prompt: str
|
||||
time_range: str = "2024 年至今"
|
||||
audience: str = "企业战略决策人员"
|
||||
required_angles: list[str]
|
||||
key_facts: list[KeyFact]
|
||||
forbidden_fabrications: list[ForbiddenItem]
|
||||
template: TemplateSpec
|
||||
review_rules: list[str]
|
||||
acceptable_sources: list[AcceptableSource]
|
||||
directed_modifications: list[DirectedModification] = Field(default_factory=list)
|
||||
|
||||
|
||||
class MetricScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
score: float = Field(ge=0, le=1)
|
||||
passed: bool
|
||||
issues: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class TaskScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
task_id: str
|
||||
source: str
|
||||
metrics: list[MetricScore]
|
||||
passed: bool
|
||||
composite: float
|
||||
|
||||
def metric(self, name: str) -> MetricScore:
|
||||
for item in self.metrics:
|
||||
if item.name == name:
|
||||
return item
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
class ComparisonResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
task_id: str
|
||||
winner: Literal["collaboration", "roundtable", "tie"]
|
||||
collaboration: TaskScore
|
||||
roundtable: TaskScore
|
||||
deltas: dict[str, float]
|
||||
notes: str
|
||||
|
||||
|
||||
class ReliabilityCounts(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
runs: int
|
||||
early_stop: int = 0
|
||||
empty_output: int = 0
|
||||
illegal_json: int = 0
|
||||
missing_events: int = 0
|
||||
completed: int = 0
|
||||
|
||||
@property
|
||||
def failure_rate(self) -> float:
|
||||
if self.runs <= 0:
|
||||
return 1.0
|
||||
failed = self.early_stop + self.empty_output + self.illegal_json + self.missing_events
|
||||
return failed / self.runs
|
||||
|
||||
|
||||
class CatalogScore(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
tasks: int
|
||||
passed: int
|
||||
scores: list[TaskScore]
|
||||
composite: float
|
||||
by_category: dict[str, float]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AcceptableSource",
|
||||
"CatalogScore",
|
||||
"ComparisonResult",
|
||||
"DirectedModification",
|
||||
"EvalCategory",
|
||||
"EvalTask",
|
||||
"FailureKind",
|
||||
"ForbiddenItem",
|
||||
"KeyFact",
|
||||
"MetricScore",
|
||||
"ReliabilityCounts",
|
||||
"ReportCandidate",
|
||||
"TaskScore",
|
||||
"TemplateSpec",
|
||||
"UsageSnapshot",
|
||||
]
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Deterministic quality scorer for annotated eval tasks. No LLM judging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.report_collaboration.eval.plan import eval_quality_plan
|
||||
from app.report_collaboration.eval.schema import (
|
||||
CatalogScore,
|
||||
DirectedModification,
|
||||
EvalTask,
|
||||
MetricScore,
|
||||
ReportCandidate,
|
||||
TaskScore,
|
||||
)
|
||||
from app.report_collaboration.eval.thresholds import METRIC_GATES, REQUIRED_EVENTS, USAGE_CAPS
|
||||
from app.report_collaboration.interventions.impact_analyzer import ImpactAnalyzer
|
||||
from app.report_collaboration.interventions.intent_router import RuntimeIntentContext, RuntimeIntentRouterAgent
|
||||
from app.report_collaboration.reporting.linter import lint_report
|
||||
from app.report_collaboration.reporting.templates import ReportTemplateSnapshot, TemplateSection
|
||||
from deerflow.persistence.report_collaboration.codec import iso_now
|
||||
|
||||
|
||||
def score_report(
|
||||
task: EvalTask,
|
||||
report: ReportCandidate,
|
||||
*,
|
||||
directed_hits: list[bool] | None = None,
|
||||
skip: set[str] | None = None,
|
||||
) -> TaskScore:
|
||||
skipped = skip or set()
|
||||
builders = (
|
||||
("factual_accuracy", lambda: _factual(task, report)),
|
||||
("source_quality", lambda: _source_quality(task, report)),
|
||||
("citation_coverage", lambda: _citation(task, report)),
|
||||
("angle_coverage", lambda: _angles(task, report)),
|
||||
("logical_consistency", lambda: _logic(task, report)),
|
||||
("template_adherence", lambda: _template(task, report)),
|
||||
("directed_modification", lambda: _directed(directed_hits)),
|
||||
("time_cost", lambda: _time_cost(report)),
|
||||
)
|
||||
kept = [build() for name, build in builders if name not in skipped]
|
||||
passed = all(item.passed for item in kept)
|
||||
composite = sum(item.score for item in kept) / max(1, len(kept))
|
||||
return TaskScore(task_id=task.id, source=report.source, metrics=kept, passed=passed, composite=round(composite, 4))
|
||||
|
||||
|
||||
async def score_directed_modifications(task: EvalTask) -> list[bool]:
|
||||
router = RuntimeIntentRouterAgent()
|
||||
analyzer = ImpactAnalyzer()
|
||||
plan = eval_quality_plan()
|
||||
hits: list[bool] = []
|
||||
for item in task.directed_modifications:
|
||||
decision = await router.route(
|
||||
RuntimeIntentContext(text=item.command, requirement={"topic": task.title, "required_angles": task.required_angles, "revision": 1}, plan_nodes=list(plan["nodes"]))
|
||||
)
|
||||
impact = analyzer.analyze(decision, plan=plan, node_runs=[], agent_runs=[], explicit_target_node_ids=list(decision.requested_target_ids))
|
||||
hits.append(_directed_hit(item, decision.intent, plan, impact.target_node_ids, impact.impact_nodes))
|
||||
return hits
|
||||
|
||||
|
||||
async def evaluate_task(task: EvalTask, report: ReportCandidate, *, skip: set[str] | None = None) -> TaskScore:
|
||||
hits = await score_directed_modifications(task)
|
||||
return score_report(task, report, directed_hits=hits, skip=skip)
|
||||
|
||||
|
||||
async def evaluate_catalog(tasks: list[EvalTask] | None = None) -> CatalogScore:
|
||||
from app.report_collaboration.eval.catalog import load_eval_tasks
|
||||
from app.report_collaboration.eval.materialize import materialize_gold
|
||||
|
||||
catalog = tasks or load_eval_tasks()
|
||||
scores = [await evaluate_task(task, materialize_gold(task)) for task in catalog]
|
||||
by_category: dict[str, list[float]] = {}
|
||||
for task, score in zip(catalog, scores, strict=True):
|
||||
by_category.setdefault(task.category, []).append(score.composite)
|
||||
return CatalogScore(
|
||||
tasks=len(scores),
|
||||
passed=sum(1 for item in scores if item.passed),
|
||||
scores=scores,
|
||||
composite=round(sum(item.composite for item in scores) / max(1, len(scores)), 4),
|
||||
by_category={key: round(sum(values) / len(values), 4) for key, values in by_category.items()},
|
||||
)
|
||||
|
||||
|
||||
def _factual(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
issues: list[str] = []
|
||||
hits = 0
|
||||
for fact in task.key_facts:
|
||||
present = fact.number in report.markdown and (fact.text in report.markdown or fact.number in report.markdown)
|
||||
if present:
|
||||
hits += 1
|
||||
else:
|
||||
issues.append(f"缺少关键事实 {fact.id}")
|
||||
for item in task.forbidden_fabrications:
|
||||
if item.text and item.text in report.markdown:
|
||||
issues.append(f"出现禁止编造项:{item.text}")
|
||||
total = max(1, len(task.key_facts))
|
||||
score = 0.0 if any("禁止编造" in issue for issue in issues) else hits / total
|
||||
return _metric("factual_accuracy", score, issues)
|
||||
|
||||
|
||||
def _source_quality(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
issues: list[str] = []
|
||||
if not report.source_index:
|
||||
return _metric("source_quality", 0.0, ["没有来源索引"])
|
||||
accepted = 0
|
||||
for source in report.source_index:
|
||||
blob = " ".join(str(source.get(key) or "") for key in ("title", "url", "id"))
|
||||
year = _year_of(str(source.get("published_at") or ""))
|
||||
matched = False
|
||||
for rule in task.acceptable_sources:
|
||||
tokens_ok = any(token in blob for token in rule.title_tokens)
|
||||
domain_ok = (rule.domain_suffix is None) or (rule.domain_suffix in str(source.get("url") or ""))
|
||||
year_ok = rule.min_year is None or year is None or year >= rule.min_year
|
||||
if tokens_ok and domain_ok and year_ok:
|
||||
matched = True
|
||||
break
|
||||
if year is not None and rule.min_year is not None and year < rule.min_year:
|
||||
issues.append(f"来源过期:{source.get('title')}")
|
||||
if matched:
|
||||
accepted += 1
|
||||
else:
|
||||
issues.append(f"来源不在可接受清单:{source.get('title')}")
|
||||
score = accepted / max(1, len(report.source_index))
|
||||
return _metric("source_quality", score, issues)
|
||||
|
||||
|
||||
def _citation(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
issues: list[str] = []
|
||||
cited_ids = {str(item.get("id") or "") for item in report.source_index if item.get("id")}
|
||||
must = [fact for fact in task.key_facts if fact.must_cite]
|
||||
hits = 0
|
||||
for fact in must:
|
||||
mentioned = fact.number in report.markdown or fact.text in report.markdown
|
||||
cited = fact.id in report.markdown or fact.id in cited_ids
|
||||
if mentioned and cited:
|
||||
hits += 1
|
||||
else:
|
||||
issues.append(f"关键主张 {fact.id} 缺少引用")
|
||||
score = hits / max(1, len(must))
|
||||
return _metric("citation_coverage", score, issues)
|
||||
|
||||
|
||||
def _angles(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
missing = [angle for angle in task.required_angles if angle and angle not in report.markdown]
|
||||
score = 1.0 - (len(missing) / max(1, len(task.required_angles)))
|
||||
return _metric("angle_coverage", score, [f"未覆盖必选角度 {angle}" for angle in missing])
|
||||
|
||||
|
||||
def _logic(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
issues = [f"正文同时写入禁止项,逻辑不可用:{item.text}" for item in task.forbidden_fabrications if item.text in report.markdown]
|
||||
return _metric("logical_consistency", 0.0 if issues else 1.0, issues)
|
||||
|
||||
|
||||
def _template(task: EvalTask, report: ReportCandidate) -> MetricScore:
|
||||
snapshot = ReportTemplateSnapshot(
|
||||
structure_id=task.id,
|
||||
title=task.template.title,
|
||||
content="\n".join(f"# {title}" for title in task.template.sections),
|
||||
content_sha256="eval",
|
||||
sections=[
|
||||
TemplateSection(
|
||||
id=f"s{index}",
|
||||
level=1,
|
||||
title=title,
|
||||
required_questions=list(task.template.required_questions) if index == 0 else [],
|
||||
required_tables=list(task.template.required_tables) if index == 0 else [],
|
||||
comparison_dimensions=list(task.template.comparison_dimensions) if index == 0 else [],
|
||||
)
|
||||
for index, title in enumerate(task.template.sections)
|
||||
],
|
||||
length_target=task.template.length_target,
|
||||
frozen_at=iso_now(),
|
||||
)
|
||||
lint = lint_report(
|
||||
report.markdown,
|
||||
template_snapshot=snapshot.model_dump(mode="json"),
|
||||
source_index=list(report.source_index),
|
||||
required_angles=list(task.required_angles),
|
||||
)
|
||||
score = 1.0 if lint.ok else max(0.0, 1.0 - 0.25 * len(lint.issues))
|
||||
return _metric("template_adherence", score, [f"{item.get('code')}: {item.get('message')}" for item in lint.issues])
|
||||
|
||||
|
||||
def _directed(hits: list[bool] | None) -> MetricScore:
|
||||
if hits is None:
|
||||
return _metric("directed_modification", 1.0, [])
|
||||
if not hits:
|
||||
return _metric("directed_modification", 1.0, [])
|
||||
score = sum(1 for item in hits if item) / len(hits)
|
||||
issues = [] if score >= METRIC_GATES["directed_modification"] else ["定向修改未按角色安全点生效"]
|
||||
return _metric("directed_modification", score, issues)
|
||||
|
||||
|
||||
def _time_cost(report: ReportCandidate) -> MetricScore:
|
||||
issues: list[str] = []
|
||||
if report.usage is None:
|
||||
if report.status != "completed":
|
||||
issues.append(f"运行未完成:{report.status}")
|
||||
missing = [name for name in REQUIRED_EVENTS if name not in set(report.events_present)]
|
||||
if report.events_present and missing:
|
||||
issues.append("缺少必要事件:" + ",".join(missing))
|
||||
return _metric("time_cost", 0.0 if issues else 1.0, issues)
|
||||
usage = report.usage
|
||||
checks = {
|
||||
"duration_seconds": usage.duration_seconds,
|
||||
"model_calls": usage.model_calls,
|
||||
"retrieval_calls": usage.retrieval_calls,
|
||||
"input_tokens": usage.input_tokens,
|
||||
"output_tokens": usage.output_tokens,
|
||||
}
|
||||
for key, value in checks.items():
|
||||
cap = USAGE_CAPS[key]
|
||||
if value > cap:
|
||||
issues.append(f"{key} {value} 超过评测上限 {cap}")
|
||||
if USAGE_CAPS["cost"] > 0 and usage.cost > USAGE_CAPS["cost"]:
|
||||
issues.append(f"cost {usage.cost} 超过评测上限")
|
||||
return _metric("time_cost", 0.0 if issues else 1.0, issues)
|
||||
|
||||
|
||||
def _directed_hit(item: DirectedModification, intent: str, plan: dict[str, Any], roots: list[str], impact: list[str]) -> bool:
|
||||
if intent != item.intent:
|
||||
return False
|
||||
nodes = {str(node["id"]): node for node in plan.get("nodes") or []}
|
||||
root_roles = {str(nodes[node_id].get("role_key") or "") for node_id in roots if node_id in nodes}
|
||||
impact_roles = {str(nodes[node_id].get("role_key") or "") for node_id in impact if node_id in nodes}
|
||||
must = set(item.must_reopen_roles)
|
||||
if not must <= (root_roles | impact_roles):
|
||||
return False
|
||||
if set(item.must_not_reopen_roles) & root_roles:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _metric(name: str, score: float, issues: list[str]) -> MetricScore:
|
||||
bounded = max(0.0, min(1.0, score))
|
||||
return MetricScore(name=name, score=round(bounded, 4), passed=bounded + 1e-9 >= METRIC_GATES[name], issues=issues)
|
||||
|
||||
|
||||
def _year_of(value: str) -> int | None:
|
||||
digits = "".join(ch for ch in value if ch.isdigit())
|
||||
if len(digits) >= 4:
|
||||
year = int(digits[:4])
|
||||
if 1900 <= year <= 2100:
|
||||
return year
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["evaluate_catalog", "evaluate_task", "score_directed_modifications", "score_report"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Pass gates and the runtime profile that the 20-task set was tuned against.
|
||||
|
||||
These numbers are the launch bar from the backend plan §11.2. They are not
|
||||
model-judged: a canned gold report must clear them, and a known failure must not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.config.report_collaboration_config import ReportCollaborationConfig
|
||||
|
||||
METRIC_GATES: dict[str, float] = {
|
||||
"factual_accuracy": 0.85,
|
||||
"source_quality": 0.80,
|
||||
"citation_coverage": 0.85,
|
||||
"angle_coverage": 1.00,
|
||||
"logical_consistency": 1.00,
|
||||
"template_adherence": 1.00,
|
||||
"directed_modification": 0.80,
|
||||
"time_cost": 1.00,
|
||||
}
|
||||
|
||||
# Production knobs that keep the gold set green. Changing a default requires
|
||||
# re-running `tests/test_report_collaboration_eval.py` and updating this table.
|
||||
RECOMMENDED_RUNTIME_PROFILE: dict[str, object] = {
|
||||
"enabled": False,
|
||||
"worker_enabled": False,
|
||||
"max_team_members": 8,
|
||||
"max_parallel_tasks": 4,
|
||||
"max_repair_rounds": 3,
|
||||
"max_node_attempts": 3,
|
||||
"max_react_iterations": 8,
|
||||
"source_policy": "cite_required",
|
||||
"token_budget": 250_000,
|
||||
"max_model_calls": 200,
|
||||
"max_retrieval_calls": 40,
|
||||
"max_source_bytes": 2_000_000,
|
||||
"max_concurrent_runs_per_user": 2,
|
||||
"intent_confirmation_threshold": 0.8,
|
||||
}
|
||||
|
||||
USAGE_CAPS = {
|
||||
"duration_seconds": 7200.0,
|
||||
"model_calls": 200,
|
||||
"retrieval_calls": 40,
|
||||
"input_tokens": 200_000,
|
||||
"output_tokens": 50_000,
|
||||
"cost": 0.0,
|
||||
}
|
||||
|
||||
REQUIRED_EVENTS = ("message.created", "report.version.created")
|
||||
|
||||
|
||||
def recommended_config() -> ReportCollaborationConfig:
|
||||
return ReportCollaborationConfig.model_validate(
|
||||
{key: value for key, value in RECOMMENDED_RUNTIME_PROFILE.items() if key in ReportCollaborationConfig.model_fields}
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["METRIC_GATES", "RECOMMENDED_RUNTIME_PROFILE", "REQUIRED_EVENTS", "USAGE_CAPS", "recommended_config"]
|
||||
Reference in new issue
Block a user