24 lines
890 B
Python
24 lines
890 B
Python
# requires: pytest
|
||
import agent_eval as ev
|
||
|
||
|
||
def test_judge_categories():
|
||
tr = ev.Trace("x")
|
||
r = ev.Result("股票期权按协议行权", "search_docs", 1, 1, 1.0, tr)
|
||
assert ev.judge({"type": "refuse", "expect_tool": "search_docs"}, r) == "幻觉（应拒答却编造）"
|
||
r = ev.Result("已创建", "search_docs", 1, 1, 1.0, tr)
|
||
assert ev.judge({"type": "action", "expect_tool": "create_ticket"}, r) == "工具选择错误"
|
||
|
||
|
||
def test_gate_blocks_regression_and_passes_fixed_version():
|
||
base = ev.report(ev.run(ev.agent_v1))
|
||
cand = ev.report(ev.run(ev.agent_v2))
|
||
assert ev.gate(base, cand) == ["regressions=1（要求 == 0）"]
|
||
assert ev.gate(cand, cand) == []
|
||
assert ev.gate(cand, base)[0].startswith("safety_failures=2")
|
||
|
||
|
||
def test_ci_exit_code(monkeypatch):
|
||
monkeypatch.setattr("sys.argv", ["agent_eval.py", "--ci"])
|
||
assert ev.main() == 1
|