初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,375 @@
from __future__ import annotations
import json
from datetime import date
import httpx
import pytest
from deerflow.community.configurable_search.tools import (
ConfigurableSearchSettings,
execute_search,
)
from deerflow.config.app_config import AppConfig
from deerflow.config.tool_config import ToolConfig
from deerflow.tools.tools import get_available_tools
def _settings(**overrides: object) -> ConfigurableSearchSettings:
values = {
"enabled": True,
"endpoint": "https://search.example.test/searchName",
"payload": {"query": "configured value", "xxx": "xxxx"},
"verify_ssl": False,
"timeout": 30.0,
"max_results": 5,
"result_url_template": "https://search.example.test/mindId/xxx?recUuid={recUuid}",
}
values.update(overrides)
return ConfigurableSearchSettings(**values) # type: ignore[arg-type]
@pytest.mark.asyncio
async def test_configurable_search_overrides_query_and_normalizes_nullable_results() -> None:
requests: list[httpx.Request] = []
async def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.method == "POST"
payload = json.loads(request.content)
assert payload["query"].strip()
assert payload["xxx"] == "xxxx"
assert payload["start_date"] == ""
assert payload["end_date"] == ""
assert payload["top_k"] == ""
return httpx.Response(
200,
json={
"results": [
{
"recUuid": "record/123",
"content": "test content",
"title": "test title",
"url": "https://original.example.test/one",
},
{
"recUuid": None,
"content": None,
"title": None,
"url": "https://original.example.test/two",
},
"invalid row",
]
},
)
settings = _settings()
result = json.loads(
await execute_search(
" test query ",
settings,
transport=httpx.MockTransport(handler),
)
)
assert len(requests) == 1
assert settings.payload == {"query": "configured value", "xxx": "xxxx"}
assert result == {
"query": "test query",
"total_results": 2,
"results": [
{
"recUuid": "record/123",
"content": "test content",
"title": "test title",
"url": "https://search.example.test/mindId/xxx?recUuid=record%2F123",
},
{
"recUuid": "",
"content": "",
"title": "",
"url": "https://original.example.test/two",
},
],
}
@pytest.mark.asyncio
async def test_configurable_search_handles_empty_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"results": []})
result = json.loads(
await execute_search(
"nothing",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result == {"query": "nothing", "total_results": 0, "results": []}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("query", "expected_start", "expected_end"),
[
("近期政策", "2026-05-13", "2026-08-13"),
("近几天政策", "2026-08-06", "2026-08-13"),
("查询 2026年8月10日 到 2026-08-13 的政策", "2026-08-10", "2026-08-13"),
],
)
async def test_configurable_search_adds_date_payload(
query: str,
expected_start: str,
expected_end: str,
) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == expected_start
assert payload["end_date"] == expected_end
assert payload["top_k"] == ""
return httpx.Response(200, json={"results": []})
await execute_search(
query,
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_explicit_date_args_and_top_k_win() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == "2026-08-10"
assert payload["end_date"] == "2026-08-13"
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策",
_settings(),
start_date="2026/8/10",
end_date="2026年8月13日",
top_k=15,
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_infers_top_k_from_query() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策 返回15条",
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_accepts_root_array_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json=[
{
"recUuid": "root-array-record",
"content": "direct array content",
"title": "direct array title",
"url": "https://original.example.test/root-array",
}
],
)
result = json.loads(
await execute_search(
"array response",
_settings(result_url_template=""),
transport=httpx.MockTransport(handler),
)
)
assert result == {
"query": "array response",
"total_results": 1,
"results": [
{
"recUuid": "root-array-record",
"content": "direct array content",
"title": "direct array title",
"url": "https://original.example.test/root-array",
}
],
}
@pytest.mark.asyncio
async def test_configurable_search_normalizes_metadata_wrapped_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json=[
{
"page_content": "outer content",
"metadata": {
"m_title": "metadata title",
"content": "metadata content",
"source1": "https://source.example.test/article/1",
"m_publish": "2026-08-13",
},
}
],
)
result = json.loads(
await execute_search(
"metadata response",
_settings(result_url_template=""),
transport=httpx.MockTransport(handler),
)
)
assert result["results"] == [
{
"recUuid": "",
"content": "outer content",
"title": "metadata title",
"source": "https://source.example.test/article/1",
"time": "2026-08-13",
"publish_time": "2026-08-13",
"url": "https://source.example.test/article/1",
}
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("response_factory", "expected_error"),
[
(lambda request: httpx.Response(503, request=request), "Search API returned HTTP 503"),
(
lambda request: httpx.Response(200, content=b"not-json", request=request),
"Search API request failed: JSONDecodeError",
),
(
lambda request: httpx.Response(200, json={"results": None}, request=request),
"search API response field 'results' must be an array",
),
],
)
async def test_configurable_search_returns_safe_errors(
response_factory,
expected_error: str,
) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
return response_factory(request)
result = json.loads(
await execute_search(
"query",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result["results"] == []
assert result["total_results"] == 0
assert result["error"] == expected_error
@pytest.mark.asyncio
async def test_configurable_search_handles_timeout() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ReadTimeout("timed out", request=request)
result = json.loads(
await execute_search(
"query",
_settings(),
transport=httpx.MockTransport(handler),
)
)
assert result["error"] == "Search API request timed out"
@pytest.mark.asyncio
async def test_configurable_search_passes_verify_ssl_to_httpx(monkeypatch: pytest.MonkeyPatch) -> None:
captured: dict[str, object] = {}
class FakeAsyncClient:
def __init__(self, **kwargs: object) -> None:
captured.update(kwargs)
async def __aenter__(self):
return self
async def __aexit__(self, *_: object) -> None:
return None
async def post(self, url: str, *, json: dict[str, object]) -> httpx.Response:
request = httpx.Request("POST", url, json=json)
return httpx.Response(200, json={"results": []}, request=request)
monkeypatch.setattr(
"deerflow.community.configurable_search.tools.httpx.AsyncClient",
FakeAsyncClient,
)
await execute_search("query", _settings(verify_ssl=False))
assert captured["verify"] is False
assert captured["timeout"] == 30.0
def test_disabled_configured_tool_is_not_registered() -> None:
config = AppConfig(
sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"},
tools=[
ToolConfig(
name="web_search",
group="web",
enabled=False,
use="deerflow.community.configurable_search.tools:web_search_tool",
endpoint="https://search.example.test/searchName",
)
],
)
available = get_available_tools(
groups=["web"],
include_mcp=False,
app_config=config,
)
assert "web_search" not in {item.name for item in available}
config.tools[0].enabled = True
enabled_tools = get_available_tools(
groups=["web"],
include_mcp=False,
app_config=config,
)
assert "web_search" in {item.name for item in enabled_tools}
def test_config_validation_requires_rec_uuid_url_placeholder() -> None:
config = ToolConfig(
name="web_search",
group="web",
use="deerflow.community.configurable_search.tools:web_search_tool",
endpoint="https://search.example.test/searchName",
result_url_template="https://search.example.test/mindId/xxx",
)
with pytest.raises(ValueError, match="must contain"):
ConfigurableSearchSettings.from_tool_config(config)
@@ -0,0 +1,84 @@
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from deerflow.agents.middlewares.writing_setup_round_cap_middleware import WritingSetupRoundCapMiddleware
from deerflow.models.message_ordering import normalize_chat_payload_messages, normalize_langchain_messages
from deerflow.models.patched_openai import PatchedChatOpenAI
def _multi_turn_payload_with_late_systems(turns: int):
messages = [{"role": "system", "content": "base system"}]
for idx in range(turns):
messages.append({"role": "user", "content": f"user {idx}"})
messages.append({"role": "assistant", "content": f"assistant {idx}"})
if idx >= 9 and (idx + 1) % 10 == 0:
messages.append({"role": "system", "content": f"late summary after {idx + 1} turns"})
messages.append({"role": "system", "content": "tail forced instruction"})
return {"messages": messages}
@pytest.mark.parametrize("turns", [12, 25, 50])
def test_normalize_chat_payload_messages_merges_late_systems_after_many_turns(turns):
payload = _multi_turn_payload_with_late_systems(turns)
normalize_chat_payload_messages(payload)
roles = [message["role"] for message in payload["messages"]]
assert roles[0] == "system"
assert roles.count("system") == 1
assert roles[1:5] == ["user", "assistant", "user", "assistant"]
assert "base system" in payload["messages"][0]["content"]
assert "late summary after 10 turns" in payload["messages"][0]["content"]
assert "tail forced instruction" in payload["messages"][0]["content"]
def test_normalize_langchain_messages_moves_late_systems_before_history():
messages = [SystemMessage(content="base system")]
for idx in range(12):
messages.append(HumanMessage(content=f"user {idx}"))
messages.append(AIMessage(content=f"assistant {idx}"))
if idx == 10:
messages.append(SystemMessage(content="late compaction summary"))
normalized = normalize_langchain_messages(messages)
assert isinstance(normalized[0], SystemMessage)
assert sum(isinstance(message, SystemMessage) for message in normalized) == 1
assert normalized[0].content == "base system\n\n---\n\nlate compaction summary"
assert isinstance(normalized[1], HumanMessage)
assert normalized[1].content == "user 0"
@pytest.mark.parametrize("turns", [12, 25, 50])
def test_patched_openai_payload_has_only_leading_system_after_many_turns(turns):
model = PatchedChatOpenAI(
model="test-model",
api_key="test-key",
base_url="http://127.0.0.1:9/v1",
)
messages = [SystemMessage(content="base system")]
for idx in range(turns):
messages.append(HumanMessage(content=f"user {idx}"))
messages.append(AIMessage(content=f"assistant {idx}"))
if idx >= 9 and (idx + 1) % 10 == 0:
messages.append(SystemMessage(content=f"late system after {idx + 1} turns"))
messages.append(SystemMessage(content="tail system"))
payload = model._get_request_payload(messages)
roles = [message["role"] for message in payload["messages"]]
assert roles[0] == "system"
assert roles.count("system") == 1
assert "base system" in payload["messages"][0]["content"]
assert "late system after 10 turns" in payload["messages"][0]["content"]
assert "tail system" in payload["messages"][0]["content"]
def test_writing_round_cap_reminder_is_not_late_system_message():
middleware = WritingSetupRoundCapMiddleware(max_rounds=3)
reminder = middleware._reminder()
assert isinstance(reminder, HumanMessage)
assert not isinstance(reminder, SystemMessage)
assert reminder.additional_kwargs["hide_from_ui"] is True
@@ -0,0 +1,96 @@
"""Tests for RAG citation batch construction."""
from __future__ import annotations
import json
from deerflow.runtime.references import (
build_latest_reference_batch,
build_latest_reference_prompt,
build_reference_batches,
)
def _tool_message(call_id: str, query: str, start: int) -> dict:
return {
"type": "tool",
"name": "web_search",
"tool_call_id": call_id,
"content": json.dumps(
{
"query": query,
"results": [
{
"title": f"{query} result {start}",
"content": f"{query} content {start}",
"url": f"https://example.test/{start}",
},
{
"title": f"{query} result {start + 1}",
"content": f"{query} content {start + 1}",
"url": f"https://example.test/{start + 1}",
},
],
},
ensure_ascii=False,
),
}
def test_latest_reference_prompt_uses_one_combined_batch_for_multi_search_turn() -> None:
messages = [
{"type": "human", "id": "human-1", "content": "search two terms"},
{
"type": "ai",
"id": "ai-search-1",
"content": "",
"tool_calls": [{"id": "call-1", "name": "web_search", "args": {"query": "Trump"}}],
},
_tool_message("call-1", "Trump", 1),
{
"type": "ai",
"id": "ai-search-2",
"content": "",
"tool_calls": [{"id": "call-2", "name": "web_search", "args": {"query": "White House"}}],
},
_tool_message("call-2", "White House", 3),
]
batch = build_latest_reference_batch(messages)
assert batch is not None
assert batch["source_count"] == 4
assert [source["index"] for source in batch["sources"]] == [1, 2, 3, 4]
assert [source["title"] for source in batch["sources"]] == [
"Trump result 1",
"Trump result 2",
"White House result 3",
"White House result 4",
]
prompt, prompt_batch = build_latest_reference_prompt(messages)
assert prompt_batch == batch
assert '<reference_batch source_count="4">' in prompt
assert "[1] Trump result 1" in prompt
assert "[4] White House result 4" in prompt
assert "重新分批" in prompt
def test_reference_batches_keep_separate_user_turns_but_merge_searches_inside_turn() -> None:
messages = [
{"type": "human", "id": "human-1", "content": "first"},
{"type": "ai", "id": "ai-1", "content": "", "tool_calls": [{"id": "c1", "name": "web_search"}]},
_tool_message("c1", "first", 1),
{"type": "ai", "id": "answer-1", "content": "answer [1]"},
{"type": "human", "id": "human-2", "content": "second"},
{"type": "ai", "id": "ai-2", "content": "", "tool_calls": [{"id": "c2", "name": "web_search"}]},
_tool_message("c2", "second-a", 3),
{"type": "ai", "id": "ai-3", "content": "", "tool_calls": [{"id": "c3", "name": "web_search"}]},
_tool_message("c3", "second-b", 5),
{"type": "ai", "id": "answer-2", "content": "answer [1][4]"},
]
batches = build_reference_batches(messages)
assert [len(batch["sources"]) for batch in batches] == [2, 4]
assert [source["index"] for source in batches[1]["sources"]] == [1, 2, 3, 4]
assert batches[0]["assistant_message_id"] == "answer-1"
assert batches[1]["assistant_message_id"] == "answer-2"