388 lines
15 KiB
Python
388 lines
15 KiB
Python
"""会商「模型自动容错（换下一个模型）」测试。
|
|
|
|
覆盖：
|
|
- 中间件兜底/熔断 AIMessage 带可识别标记 + ``is_llm_error_message`` 检测（标记 + 正文回退）；
|
|
- ``fallback_model_chain`` 按 config.yaml models[] 顺序排候选（当前优先、去重、开关关闭=单个）；
|
|
- ``reason_text`` 文案；
|
|
- 引擎 Step2 单席位异常隔离（一个席位抛错不拖垮整链/整批，仅置空交付）；
|
|
- 网关带回的模型容错记录（switches）→ 引擎在时间线插「系统提示」气泡。
|
|
|
|
纯单元/纯逻辑：引擎用 JobProgressWriter(store=None) 干跑 + 假网关，不需要 DB / 真模型。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from contextlib import asynccontextmanager
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import app.gateway.roundtable_model_fallback as mf
|
|
import app.gateway.routers.parallel_agents as parallel_agents
|
|
from app.gateway.roundtable_model_fallback import (
|
|
fallback_model_chain,
|
|
is_llm_error_message,
|
|
reason_text,
|
|
)
|
|
from deerflow.agents.middlewares.llm_error_handling_middleware import (
|
|
LLM_ERROR_MARKER,
|
|
LLMErrorHandlingMiddleware,
|
|
)
|
|
from deerflow.agents.middlewares.llm_error_handling_middleware import (
|
|
is_llm_error_message as harness_is_llm_error_message,
|
|
)
|
|
from deerflow.agents.roundtable_orchestrator import (
|
|
JobProgressWriter,
|
|
SeatRef,
|
|
run_orchestration,
|
|
)
|
|
from deerflow.agents.roundtable_orchestrator.types import LeaderTurn, ModelSwitch, SeatTurn
|
|
|
|
# ── is_llm_error_message ─────────────────────────────────────────────────────
|
|
|
|
|
|
def test_is_llm_error_message_marker_dict() -> None:
|
|
msg = {"type": "ai", "content": "随便什么", "additional_kwargs": {LLM_ERROR_MARKER: "busy"}}
|
|
assert is_llm_error_message(msg) == "busy"
|
|
assert harness_is_llm_error_message(msg) == "busy"
|
|
|
|
|
|
def test_is_llm_error_message_content_fallback() -> None:
|
|
# additional_kwargs 被剥离时，按已知兜底文案正文回退识别。
|
|
msg = {"type": "ai", "content": "大模型本次调用未能成功完成，请稍后再试或更换模型。"}
|
|
assert is_llm_error_message(msg) == "generic"
|
|
|
|
|
|
def test_is_llm_error_message_normal_is_none() -> None:
|
|
assert is_llm_error_message({"type": "ai", "content": "这是正常的席位交付内容"}) is None
|
|
assert is_llm_error_message(None) is None
|
|
assert is_llm_error_message({"type": "ai", "content": ""}) is None
|
|
|
|
|
|
def test_parallel_agent_run_raises_instead_of_completing_llm_fallback(monkeypatch) -> None:
|
|
fallback_message = {
|
|
"type": "ai",
|
|
"content": "大模型本次调用未能成功完成，请稍后再试或更换模型。",
|
|
}
|
|
|
|
async def fake_stream_upstream(*_args, **_kwargs):
|
|
yield None, [f"data: {json.dumps({'messages': [fallback_message]}, ensure_ascii=False)}"]
|
|
|
|
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
|
|
|
async def collect() -> list[object]:
|
|
return [
|
|
item
|
|
async for item in parallel_agents._run_one_agent(
|
|
None, # type: ignore[arg-type]
|
|
"http://gateway.test",
|
|
{"Authorization": "Bearer test"},
|
|
"strategy-selection-intent",
|
|
"thread-id",
|
|
"test task",
|
|
"",
|
|
)
|
|
]
|
|
|
|
try:
|
|
asyncio.run(collect())
|
|
except RuntimeError as exc:
|
|
assert "模型调用失败" in str(exc)
|
|
else:
|
|
raise AssertionError("LLM fallback message must not be emitted as agent_done")
|
|
|
|
|
|
def test_parallel_agent_run_rejects_punctuation_only_delivery(monkeypatch) -> None:
|
|
invalid_message = {"type": "ai", "content": "!" * 500}
|
|
|
|
async def fake_stream_upstream(*_args, **_kwargs):
|
|
yield None, [f"data: {json.dumps({'messages': [invalid_message]}, ensure_ascii=False)}"]
|
|
|
|
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
|
|
|
async def collect() -> list[object]:
|
|
return [
|
|
item
|
|
async for item in parallel_agents._run_one_agent(
|
|
None, # type: ignore[arg-type]
|
|
"http://gateway.test",
|
|
{"Authorization": "Bearer test"},
|
|
"strategy-selection-angle-security",
|
|
"thread-id",
|
|
"test task",
|
|
"",
|
|
)
|
|
]
|
|
|
|
try:
|
|
asyncio.run(collect())
|
|
except RuntimeError as exc:
|
|
assert "未生成有效正文" in str(exc)
|
|
else:
|
|
raise AssertionError("Punctuation-only output must not be emitted as agent_done")
|
|
|
|
|
|
def test_parallel_agent_run_allows_empty_content_clarification(monkeypatch) -> None:
|
|
clarification_message = {
|
|
"type": "ai",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "clarify-1",
|
|
"name": "ask_clarification",
|
|
"args": {
|
|
"question": "请确认策略时间范围。",
|
|
"clarification_type": "single_choice",
|
|
"options": ["短期", "长期"],
|
|
"allow_custom": True,
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
async def fake_stream_upstream(*_args, **_kwargs):
|
|
yield None, [
|
|
f"data: {json.dumps({'messages': [clarification_message]}, ensure_ascii=False)}"
|
|
]
|
|
|
|
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
|
|
|
async def collect() -> list[object]:
|
|
return [
|
|
item
|
|
async for item in parallel_agents._run_one_agent(
|
|
None, # type: ignore[arg-type]
|
|
"http://gateway.test",
|
|
{"Authorization": "Bearer test"},
|
|
"strategy-selection-intent",
|
|
"thread-id",
|
|
"test task",
|
|
"",
|
|
)
|
|
]
|
|
|
|
events = asyncio.run(collect())
|
|
|
|
assert len(events) == 1
|
|
event = events[0]
|
|
assert isinstance(event, dict)
|
|
assert event["type"] == "clarification"
|
|
assert event["content"] == ""
|
|
assert event["clarification"]["question"] == "请确认策略时间范围。"
|
|
|
|
|
|
def test_parallel_agent_substantive_content_accepts_chinese_and_english() -> None:
|
|
assert parallel_agents._has_substantive_content("这是一份包含目标、前置条件、阶段路径和验收标准的完整策略。")
|
|
assert parallel_agents._has_substantive_content(
|
|
"A complete strategy with objectives, prerequisites, milestones, and measurable outcomes."
|
|
)
|
|
assert not parallel_agents._has_substantive_content("<think>internal reasoning only</think>!!!")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("buffered_content", "expected_error"),
|
|
[
|
|
("大模型本次调用未能成功完成，请稍后再试或更换模型。", "模型调用失败"),
|
|
("", "未生成有效正文"),
|
|
("<think>internal reasoning only</think>", "未生成有效正文"),
|
|
("!" * 500, "未生成有效正文"),
|
|
],
|
|
)
|
|
def test_parallel_agent_truncated_stream_rejects_invalid_buffer(
|
|
monkeypatch,
|
|
buffered_content: str,
|
|
expected_error: str,
|
|
) -> None:
|
|
async def fake_run_one_agent(*_args, **_kwargs):
|
|
yield "event: messages"
|
|
yield f"data: {json.dumps([{'type': 'ai', 'content': buffered_content}], ensure_ascii=False)}"
|
|
|
|
@asynccontextmanager
|
|
async def fake_client(*_args, **_kwargs):
|
|
yield object()
|
|
|
|
async def fake_create_thread(*_args, **_kwargs):
|
|
return "thread-id"
|
|
|
|
monkeypatch.setattr(parallel_agents, "_run_one_agent", fake_run_one_agent)
|
|
monkeypatch.setattr(parallel_agents, "_make_loopback_client", fake_client)
|
|
monkeypatch.setattr(parallel_agents, "_create_thread", fake_create_thread)
|
|
monkeypatch.setattr(parallel_agents, "_loopback_base", lambda _request: "http://gateway.test")
|
|
monkeypatch.setattr(parallel_agents, "_auth_headers", lambda _request: {})
|
|
|
|
async def collect() -> list[dict[str, object]]:
|
|
response = await parallel_agents.run_parallel_agents_stream(
|
|
SimpleNamespace(),
|
|
{
|
|
"task_name": "test task",
|
|
"brief": "test brief",
|
|
"agent_ids": ["strategy-selection-intent"],
|
|
},
|
|
)
|
|
events: list[dict[str, object]] = []
|
|
async for chunk in response.body_iterator:
|
|
text = chunk.decode() if isinstance(chunk, bytes) else chunk
|
|
for line in text.splitlines():
|
|
if line.startswith("data:"):
|
|
events.append(json.loads(line[5:].strip()))
|
|
return events
|
|
|
|
events = asyncio.run(collect())
|
|
assert not any(event.get("type") == "agent_done" for event in events)
|
|
errors = [event for event in events if event.get("type") == "agent_error"]
|
|
assert len(errors) == 1
|
|
assert expected_error in str(errors[0].get("message"))
|
|
|
|
|
|
# ── 中间件给兜底 AIMessage 打标记 ────────────────────────────────────────────
|
|
|
|
|
|
def _middleware() -> LLMErrorHandlingMiddleware:
|
|
app_config = SimpleNamespace(circuit_breaker=SimpleNamespace(failure_threshold=5, recovery_timeout_sec=30))
|
|
return LLMErrorHandlingMiddleware(app_config=app_config)
|
|
|
|
|
|
def test_middleware_marks_error_message() -> None:
|
|
mw = _middleware()
|
|
|
|
def _boom(_req):
|
|
raise RuntimeError("insufficient_quota: 余额不足") # quota → 非重试 → 立即兜底
|
|
|
|
out = mw.wrap_model_call(None, _boom)
|
|
assert harness_is_llm_error_message(out) == "quota"
|
|
assert out.additional_kwargs.get(LLM_ERROR_MARKER) == "quota"
|
|
|
|
|
|
# ── fallback_model_chain ─────────────────────────────────────────────────────
|
|
|
|
|
|
def _patch_models(monkeypatch, names: list[str]) -> None:
|
|
cfg = SimpleNamespace(models=[SimpleNamespace(name=n) for n in names])
|
|
monkeypatch.setattr(mf, "get_app_config", lambda: cfg)
|
|
|
|
|
|
def test_chain_current_first_then_config_order(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, ["m1", "m2", "m3"])
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK", raising=False)
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK_MAX", raising=False)
|
|
assert fallback_model_chain("m2") == ["m2", "m1", "m3"]
|
|
|
|
|
|
def test_chain_empty_current_uses_config_order(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, ["m1", "m2", "m3"])
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK", raising=False)
|
|
assert fallback_model_chain("") == ["m1", "m2", "m3"]
|
|
assert fallback_model_chain(None) == ["m1", "m2", "m3"]
|
|
|
|
|
|
def test_chain_unknown_current_kept_first(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, ["m1", "m2"])
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK", raising=False)
|
|
assert fallback_model_chain("custom") == ["custom", "m1", "m2"]
|
|
|
|
|
|
def test_chain_disabled_single(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, ["m1", "m2", "m3"])
|
|
monkeypatch.setenv("ROUNDTABLE_MODEL_FALLBACK", "0")
|
|
assert fallback_model_chain("m2") == ["m2"]
|
|
|
|
|
|
def test_chain_cap(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, ["m1", "m2", "m3", "m4"])
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK", raising=False)
|
|
monkeypatch.setenv("ROUNDTABLE_MODEL_FALLBACK_MAX", "2")
|
|
assert fallback_model_chain("m1") == ["m1", "m2"]
|
|
|
|
|
|
def test_chain_no_config_returns_current(monkeypatch) -> None:
|
|
_patch_models(monkeypatch, [])
|
|
monkeypatch.delenv("ROUNDTABLE_MODEL_FALLBACK", raising=False)
|
|
assert fallback_model_chain("m9") == ["m9"]
|
|
assert fallback_model_chain("") == []
|
|
|
|
|
|
def test_reason_text() -> None:
|
|
assert reason_text("busy") == "服务繁忙"
|
|
assert reason_text("empty") == "未产出有效内容"
|
|
assert reason_text("timeout") == "长时间无响应（卡死）"
|
|
assert reason_text(None) == "调用失败"
|
|
assert reason_text("不存在的码") == "调用失败"
|
|
|
|
|
|
# ── 引擎：单席位异常隔离 + 模型容错记录 → 系统提示气泡 ──────────────────────────
|
|
|
|
_AGENTS = [SeatRef("seat-a", "情报"), SeatRef("seat-b", "环境"), SeatRef("seat-c", "方案")]
|
|
|
|
|
|
def _writer() -> JobProgressWriter:
|
|
return JobProgressWriter(None, job_id="job-x", agents=_AGENTS)
|
|
|
|
|
|
def _thread_ids() -> dict[str, str]:
|
|
return {a.agent_id: f"tid-{a.agent_id}" for a in _AGENTS} | {"roundtable-coordinator": "tid-co"}
|
|
|
|
|
|
class _ChainGateway:
|
|
"""chain 模式假网关：可指定某席位抛异常 / 带回模型容错记录。"""
|
|
|
|
def __init__(self, *, fail_agent: str | None = None, switch_agent: str | None = None) -> None:
|
|
self.fail_agent = fail_agent
|
|
self.switch_agent = switch_agent
|
|
self.seen: list[str] = []
|
|
|
|
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
|
|
return LeaderTurn(final_text="总控说明", dispatched=[])
|
|
|
|
async def run_seat(self, *, thread_ids, agent_id, task, model) -> SeatTurn:
|
|
self.seen.append(agent_id)
|
|
if agent_id == self.fail_agent:
|
|
raise RuntimeError("boom: 模型连接被打满")
|
|
switches = []
|
|
if agent_id == self.switch_agent:
|
|
switches = [ModelSwitch(failed_model="m1", next_model="m2", reason_label="服务繁忙")]
|
|
return SeatTurn(final_text=f"{agent_id} 的交付", switches=switches)
|
|
|
|
async def create_report_thread(self) -> str: # pragma: no cover
|
|
raise AssertionError("enable_report=False 不应绘制报告")
|
|
|
|
|
|
async def _run_chain(gw: _ChainGateway) -> tuple[object, JobProgressWriter]:
|
|
writer = _writer()
|
|
result = await run_orchestration(
|
|
gateway=gw,
|
|
progress=writer,
|
|
agents=_AGENTS,
|
|
thread_ids=_thread_ids(),
|
|
coordinator_name="roundtable-coordinator",
|
|
model="m1",
|
|
seed_message="种子任务",
|
|
mode="chain",
|
|
enable_report=False,
|
|
)
|
|
return result, writer
|
|
|
|
|
|
def test_chain_seat_failure_isolated() -> None:
|
|
gw = _ChainGateway(fail_agent="seat-b")
|
|
result, writer = asyncio.run(_run_chain(gw))
|
|
# 单席位失败不拖垮整链：仍走完三个席位并收口为 done。
|
|
assert result.status == "done"
|
|
assert gw.seen == ["seat-a", "seat-b", "seat-c"]
|
|
# 失败席位的交付气泡为空，其余正常。
|
|
seat_dlgs = {d["agentId"]: d["content"] for d in writer.dialogues() if d["role"] == "seat"}
|
|
assert seat_dlgs["seat-b"] == ""
|
|
assert seat_dlgs["seat-a"] == "seat-a 的交付"
|
|
assert seat_dlgs["seat-c"] == "seat-c 的交付"
|
|
|
|
|
|
def test_model_switch_emits_system_notice() -> None:
|
|
gw = _ChainGateway(switch_agent="seat-b")
|
|
_result, writer = asyncio.run(_run_chain(gw))
|
|
systems = [d for d in writer.dialogues() if d["role"] == "system"]
|
|
assert len(systems) == 1
|
|
assert "模型容错" in systems[0]["content"]
|
|
assert "服务繁忙" in systems[0]["content"]
|
|
assert "m2" in systems[0]["content"]
|