feat: add unified strategy project workflow
This commit is contained in:
1 parent
67d12929fc
commit
a26fd32eba
28 files changed
+3382
-191
No files matched your search
@@ -11,9 +11,11 @@ import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
|
||||
from deerflow.agents.middlewares.llm_error_handling_middleware import LLM_ERROR_MARKER, LLMErrorHandlingMiddleware
|
||||
from deerflow.models.patched_openai import OpenAICompatibleUpstreamError, PatchedChatOpenAI
|
||||
|
||||
|
||||
def _middleware() -> LLMErrorHandlingMiddleware:
|
||||
@@ -26,6 +28,10 @@ def _middleware() -> LLMErrorHandlingMiddleware:
|
||||
return middleware
|
||||
|
||||
|
||||
def _openai_model() -> PatchedChatOpenAI:
|
||||
return PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
|
||||
|
||||
def test_httpx_read_timeout_is_transient() -> None:
|
||||
assert _middleware()._classify_error(httpx.ReadTimeout("read timed out")) == (True, "transient")
|
||||
|
||||
@@ -46,6 +52,98 @@ def test_value_error_stays_generic() -> None:
|
||||
assert _middleware()._classify_error(ValueError("boom")) == (False, "generic")
|
||||
|
||||
|
||||
def test_openai_compatible_embedded_error_is_retriable_upstream() -> None:
|
||||
error = RuntimeError("OpenAI-compatible upstream application error (code=2056)")
|
||||
assert _middleware()._classify_error(error) == (True, "upstream")
|
||||
|
||||
|
||||
def test_openai_compatible_null_choices_error_is_retriable_upstream() -> None:
|
||||
error = TypeError("Received response with null value for 'choices'.")
|
||||
assert _middleware()._classify_error(error) == (True, "upstream")
|
||||
|
||||
|
||||
def test_insufficient_balance_stays_non_retriable_quota() -> None:
|
||||
"""The 200-body compatibility fix must not hide a real billing failure."""
|
||||
|
||||
class PaymentError(RuntimeError):
|
||||
status_code = 402
|
||||
|
||||
error = PaymentError("Insufficient Balance")
|
||||
assert _middleware()._classify_error(error) == (False, "quota")
|
||||
|
||||
|
||||
def test_minimax_token_plan_limit_stays_non_retriable_quota() -> None:
|
||||
error = OpenAICompatibleUpstreamError(
|
||||
code=2056,
|
||||
message="Token Plan limit reached",
|
||||
)
|
||||
|
||||
assert _middleware()._classify_error(error) == (False, "quota")
|
||||
|
||||
|
||||
def test_non_stream_embedded_application_error_is_raised() -> None:
|
||||
response = {
|
||||
"id": "failed-response",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": None,
|
||||
"base_resp": {
|
||||
"status_code": 2056,
|
||||
"status_msg": "Token Plan limit reached",
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(OpenAICompatibleUpstreamError, match=r"code=2056") as exc_info:
|
||||
_openai_model()._create_chat_result(response)
|
||||
|
||||
assert exc_info.value.code == 2056
|
||||
assert "Token Plan limit reached" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_stream_embedded_application_error_is_raised() -> None:
|
||||
chunk = {
|
||||
"id": "failed-response",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [],
|
||||
"base_resp": {
|
||||
"status_code": 2056,
|
||||
"status_msg": "Token Plan limit reached",
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(OpenAICompatibleUpstreamError, match=r"code=2056"):
|
||||
_openai_model()._convert_chunk_to_generation_chunk(chunk, AIMessageChunk, None)
|
||||
|
||||
|
||||
def test_normal_empty_usage_chunk_is_not_treated_as_error() -> None:
|
||||
chunk = {
|
||||
"id": "usage-response",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 3,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 5,
|
||||
},
|
||||
}
|
||||
|
||||
result = _openai_model()._convert_chunk_to_generation_chunk(chunk, AIMessageChunk, None)
|
||||
|
||||
assert result is not None
|
||||
assert result.message.usage_metadata == {
|
||||
"input_tokens": 3,
|
||||
"output_tokens": 2,
|
||||
"total_tokens": 5,
|
||||
"input_token_details": {},
|
||||
"output_token_details": {},
|
||||
}
|
||||
|
||||
|
||||
def test_read_timeout_is_retried_and_recovers() -> None:
|
||||
"""第一次 ReadTimeout、第二次成功:必须重试并返回真实结果。"""
|
||||
middleware = _middleware()
|
||||
|
||||
@@ -13,9 +13,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import app.gateway.roundtable_model_fallback as mf
|
||||
import app.gateway.routers.parallel_agents as parallel_agents
|
||||
from app.gateway.roundtable_model_fallback import (
|
||||
fallback_model_chain,
|
||||
is_llm_error_message,
|
||||
@@ -56,6 +61,181 @@ def test_is_llm_error_message_normal_is_none() -> None:
|
||||
assert is_llm_error_message({"type": "ai", "content": ""}) is None
|
||||
|
||||
|
||||
def test_parallel_agent_run_raises_instead_of_completing_llm_fallback(monkeypatch) -> None:
|
||||
fallback_message = {
|
||||
"type": "ai",
|
||||
"content": "大模型本次调用未能成功完成,请稍后再试或更换模型。",
|
||||
}
|
||||
|
||||
async def fake_stream_upstream(*_args, **_kwargs):
|
||||
yield None, [f"data: {json.dumps({'messages': [fallback_message]}, ensure_ascii=False)}"]
|
||||
|
||||
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
||||
|
||||
async def collect() -> list[object]:
|
||||
return [
|
||||
item
|
||||
async for item in parallel_agents._run_one_agent(
|
||||
None, # type: ignore[arg-type]
|
||||
"http://gateway.test",
|
||||
{"Authorization": "Bearer test"},
|
||||
"strategy-selection-intent",
|
||||
"thread-id",
|
||||
"test task",
|
||||
"",
|
||||
)
|
||||
]
|
||||
|
||||
try:
|
||||
asyncio.run(collect())
|
||||
except RuntimeError as exc:
|
||||
assert "模型调用失败" in str(exc)
|
||||
else:
|
||||
raise AssertionError("LLM fallback message must not be emitted as agent_done")
|
||||
|
||||
|
||||
def test_parallel_agent_run_rejects_punctuation_only_delivery(monkeypatch) -> None:
|
||||
invalid_message = {"type": "ai", "content": "!" * 500}
|
||||
|
||||
async def fake_stream_upstream(*_args, **_kwargs):
|
||||
yield None, [f"data: {json.dumps({'messages': [invalid_message]}, ensure_ascii=False)}"]
|
||||
|
||||
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
||||
|
||||
async def collect() -> list[object]:
|
||||
return [
|
||||
item
|
||||
async for item in parallel_agents._run_one_agent(
|
||||
None, # type: ignore[arg-type]
|
||||
"http://gateway.test",
|
||||
{"Authorization": "Bearer test"},
|
||||
"strategy-selection-angle-security",
|
||||
"thread-id",
|
||||
"test task",
|
||||
"",
|
||||
)
|
||||
]
|
||||
|
||||
try:
|
||||
asyncio.run(collect())
|
||||
except RuntimeError as exc:
|
||||
assert "未生成有效正文" in str(exc)
|
||||
else:
|
||||
raise AssertionError("Punctuation-only output must not be emitted as agent_done")
|
||||
|
||||
|
||||
def test_parallel_agent_run_allows_empty_content_clarification(monkeypatch) -> None:
|
||||
clarification_message = {
|
||||
"type": "ai",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "clarify-1",
|
||||
"name": "ask_clarification",
|
||||
"args": {
|
||||
"question": "请确认策略时间范围。",
|
||||
"clarification_type": "single_choice",
|
||||
"options": ["短期", "长期"],
|
||||
"allow_custom": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
async def fake_stream_upstream(*_args, **_kwargs):
|
||||
yield None, [
|
||||
f"data: {json.dumps({'messages': [clarification_message]}, ensure_ascii=False)}"
|
||||
]
|
||||
|
||||
monkeypatch.setattr(parallel_agents, "_stream_upstream", fake_stream_upstream)
|
||||
|
||||
async def collect() -> list[object]:
|
||||
return [
|
||||
item
|
||||
async for item in parallel_agents._run_one_agent(
|
||||
None, # type: ignore[arg-type]
|
||||
"http://gateway.test",
|
||||
{"Authorization": "Bearer test"},
|
||||
"strategy-selection-intent",
|
||||
"thread-id",
|
||||
"test task",
|
||||
"",
|
||||
)
|
||||
]
|
||||
|
||||
events = asyncio.run(collect())
|
||||
|
||||
assert len(events) == 1
|
||||
event = events[0]
|
||||
assert isinstance(event, dict)
|
||||
assert event["type"] == "clarification"
|
||||
assert event["content"] == ""
|
||||
assert event["clarification"]["question"] == "请确认策略时间范围。"
|
||||
|
||||
|
||||
def test_parallel_agent_substantive_content_accepts_chinese_and_english() -> None:
|
||||
assert parallel_agents._has_substantive_content("这是一份包含目标、前置条件、阶段路径和验收标准的完整策略。")
|
||||
assert parallel_agents._has_substantive_content(
|
||||
"A complete strategy with objectives, prerequisites, milestones, and measurable outcomes."
|
||||
)
|
||||
assert not parallel_agents._has_substantive_content("<think>internal reasoning only</think>!!!")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("buffered_content", "expected_error"),
|
||||
[
|
||||
("大模型本次调用未能成功完成,请稍后再试或更换模型。", "模型调用失败"),
|
||||
("", "未生成有效正文"),
|
||||
("<think>internal reasoning only</think>", "未生成有效正文"),
|
||||
("!" * 500, "未生成有效正文"),
|
||||
],
|
||||
)
|
||||
def test_parallel_agent_truncated_stream_rejects_invalid_buffer(
|
||||
monkeypatch,
|
||||
buffered_content: str,
|
||||
expected_error: str,
|
||||
) -> None:
|
||||
async def fake_run_one_agent(*_args, **_kwargs):
|
||||
yield "event: messages"
|
||||
yield f"data: {json.dumps([{'type': 'ai', 'content': buffered_content}], ensure_ascii=False)}"
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_client(*_args, **_kwargs):
|
||||
yield object()
|
||||
|
||||
async def fake_create_thread(*_args, **_kwargs):
|
||||
return "thread-id"
|
||||
|
||||
monkeypatch.setattr(parallel_agents, "_run_one_agent", fake_run_one_agent)
|
||||
monkeypatch.setattr(parallel_agents, "_make_loopback_client", fake_client)
|
||||
monkeypatch.setattr(parallel_agents, "_create_thread", fake_create_thread)
|
||||
monkeypatch.setattr(parallel_agents, "_loopback_base", lambda _request: "http://gateway.test")
|
||||
monkeypatch.setattr(parallel_agents, "_auth_headers", lambda _request: {})
|
||||
|
||||
async def collect() -> list[dict[str, object]]:
|
||||
response = await parallel_agents.run_parallel_agents_stream(
|
||||
SimpleNamespace(),
|
||||
{
|
||||
"task_name": "test task",
|
||||
"brief": "test brief",
|
||||
"agent_ids": ["strategy-selection-intent"],
|
||||
},
|
||||
)
|
||||
events: list[dict[str, object]] = []
|
||||
async for chunk in response.body_iterator:
|
||||
text = chunk.decode() if isinstance(chunk, bytes) else chunk
|
||||
for line in text.splitlines():
|
||||
if line.startswith("data:"):
|
||||
events.append(json.loads(line[5:].strip()))
|
||||
return events
|
||||
|
||||
events = asyncio.run(collect())
|
||||
assert not any(event.get("type") == "agent_done" for event in events)
|
||||
errors = [event for event in events if event.get("type") == "agent_error"]
|
||||
assert len(errors) == 1
|
||||
assert expected_error in str(errors[0].get("message"))
|
||||
|
||||
|
||||
# ── 中间件给兜底 AIMessage 打标记 ────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -6,9 +6,13 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.routers import _strategy_selection_seed as seed
|
||||
from app.gateway.routers import agents as agents_router
|
||||
from deerflow.config.paths import Paths
|
||||
from deerflow.persistence.skills.memory import MemorySkillStore
|
||||
from deerflow.skills.types import Skill, SkillCategory
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@@ -73,6 +77,18 @@ def test_seed_assets_use_role_appropriate_existing_skills_and_tools() -> None:
|
||||
assert solution["tool_groups"] == ["file:write"]
|
||||
|
||||
|
||||
def test_default_strategy_skills_are_enabled_in_the_system_catalog() -> None:
|
||||
extensions_path = Path(__file__).resolve().parents[1] / "extensions_config.json"
|
||||
extensions = yaml.safe_load(extensions_path.read_text(encoding="utf-8")) or {}
|
||||
configured = extensions.get("skills") or {}
|
||||
|
||||
# A missing entry means enabled by default. An explicit false would make
|
||||
# the seeded allowlist impossible to save under the server-side validator
|
||||
# and silently remove the skill from strategy runs.
|
||||
for skill_name in ("intent-clarification-policy", "policy-knowledge"):
|
||||
assert (configured.get(skill_name) or {}).get("enabled", True) is True
|
||||
|
||||
|
||||
def test_seed_assets_define_machine_readable_handoff_contracts() -> None:
|
||||
intent_soul = _asset(seed.STRATEGY_INTENT_AGENT_ID, "SOUL.md").read_text(encoding="utf-8")
|
||||
judge_soul = _asset(seed.STRATEGY_JUDGE_AGENT_ID, "SOUL.md").read_text(encoding="utf-8")
|
||||
@@ -113,3 +129,120 @@ def test_seed_creates_once_and_never_overwrites_admin_changes(tmp_paths: Paths)
|
||||
|
||||
assert seed.ensure_strategy_selection_agents() == []
|
||||
assert customized_soul.read_text(encoding="utf-8") == "管理员现场修改"
|
||||
|
||||
|
||||
class _SkillStorage:
|
||||
def __init__(self, skills: list[Skill]) -> None:
|
||||
self._skills = skills
|
||||
|
||||
def load_skills(self, *, enabled_only: bool) -> list[Skill]:
|
||||
if enabled_only:
|
||||
return [skill for skill in self._skills if skill.enabled]
|
||||
return list(self._skills)
|
||||
|
||||
|
||||
def _skill(name: str, *, category: SkillCategory, enabled: bool = True) -> Skill:
|
||||
directory = Path("skills") / category.value / name
|
||||
return Skill(
|
||||
name=name,
|
||||
description=name,
|
||||
license=None,
|
||||
skill_dir=directory,
|
||||
skill_file=directory / "SKILL.md",
|
||||
relative_path=Path(name),
|
||||
category=category,
|
||||
enabled=enabled,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_strategy_skill_validation_accepts_only_global_catalog_names(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
skill_store = MemorySkillStore()
|
||||
await skill_store.create(
|
||||
{
|
||||
"name": "published-custom",
|
||||
"owner_user_id": "publisher",
|
||||
"published": True,
|
||||
}
|
||||
)
|
||||
storage = _SkillStorage(
|
||||
[
|
||||
_skill("public-skill", category=SkillCategory.PUBLIC),
|
||||
_skill("published-custom", category=SkillCategory.CUSTOM),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(agents_router, "get_or_new_skill_storage", lambda **_: storage)
|
||||
monkeypatch.setattr(
|
||||
agents_router,
|
||||
"get_skill_usage_entries",
|
||||
lambda: {},
|
||||
)
|
||||
|
||||
result = await agents_router._validate_strategy_agent_skills(
|
||||
[" public-skill ", "published-custom", "public-skill"],
|
||||
app_config=object(), # type: ignore[arg-type]
|
||||
skill_store=skill_store,
|
||||
)
|
||||
|
||||
assert result == ["public-skill", "published-custom"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("requested", "detail_fragment"),
|
||||
[
|
||||
("missing", "Unknown strategy agent skills"),
|
||||
("disabled", "Disabled or archived strategy agent skills"),
|
||||
("stale", "Disabled or archived strategy agent skills"),
|
||||
("private-custom", "must be globally available"),
|
||||
("orphan-custom", "must be globally available"),
|
||||
],
|
||||
)
|
||||
async def test_strategy_skill_validation_rejects_unselectable_names(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
requested: str,
|
||||
detail_fragment: str,
|
||||
) -> None:
|
||||
skill_store = MemorySkillStore()
|
||||
await skill_store.create(
|
||||
{
|
||||
"name": "private-custom",
|
||||
"owner_user_id": "owner",
|
||||
"published": False,
|
||||
}
|
||||
)
|
||||
storage = _SkillStorage(
|
||||
[
|
||||
_skill("disabled", category=SkillCategory.PUBLIC, enabled=False),
|
||||
_skill("stale", category=SkillCategory.PUBLIC),
|
||||
_skill("private-custom", category=SkillCategory.CUSTOM),
|
||||
_skill("orphan-custom", category=SkillCategory.CUSTOM),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(agents_router, "get_or_new_skill_storage", lambda **_: storage)
|
||||
monkeypatch.setattr(
|
||||
agents_router,
|
||||
"get_skill_usage_entries",
|
||||
lambda: {"stale": {"state": "stale"}},
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await agents_router._validate_strategy_agent_skills(
|
||||
[requested],
|
||||
app_config=object(), # type: ignore[arg-type]
|
||||
skill_store=skill_store,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 422
|
||||
assert detail_fragment in str(exc_info.value.detail)
|
||||
|
||||
|
||||
def test_fixed_strategy_role_agents_cannot_be_deleted() -> None:
|
||||
for agent_id in seed.required_agent_ids():
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
agents_router._ensure_strategy_agent_deletable(agent_id)
|
||||
assert exc_info.value.status_code == 409
|
||||
|
||||
agents_router._ensure_strategy_agent_deletable("ordinary-custom-agent")
|
||||
Reference in new issue
Block a user