fix: harden WeKnora proxy and configurable search

This commit is contained in:
Administrator committed 2026-08-13 23:44:47 +08:00
1 parent b527bc94bb
commit 11e54550ed
10 files changed
+721 -72

No files matched your search

@@ -1,12 +1,13 @@
"""Configurable JSON-over-HTTP implementation of the ``web_search`` tool."""
"""Configurable JSON-over-HTTP implementation of the ``web_search`` tool."""
from __future__ import annotations
import json
import logging
import re
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime
from datetime import date, datetime, timedelta
from typing import Any
from urllib.parse import quote
from zoneinfo import ZoneInfo
@@ -19,11 +20,10 @@ from deerflow.config.tool_config import ToolConfig
logger = logging.getLogger(__name__)
_BEIJING_TZ = ZoneInfo("Asia/Shanghai")
def _current_time_hint() -> str:
now = datetime.now(_BEIJING_TZ)
return now.strftime("%Y-%m-%d %H:%M:%S, %A, Asia/Shanghai")
_ISO_DATE_RE = re.compile(r"(?P<year>\d{4})(?:[-/.]|\u5e74)(?P<month>\d{1,2})(?:[-/.]|\u6708)(?P<day>\d{1,2})(?:\u65e5)?")
_RECENT_DAYS_RE = re.compile(r"(\u8fd1\u51e0\u5929|\u6700\u8fd1\u51e0\u5929|\u8fd9\u51e0\u5929|\u8fd1\u4e00\u5468|\u6700\u8fd1\u4e00\u5468|\u8fd17\u5929|\u6700\u8fd17\u5929)")
_RECENT_MONTHS_RE = re.compile(r"(\u8fd1\u671f|\u6700\u8fd1|\u8fd1\u4e09\u4e2a\u6708|\u6700\u8fd1\u4e09\u4e2a\u6708|\u8fd13\u4e2a\u6708|\u6700\u8fd13\u4e2a\u6708)")
_TOP_K_RE = re.compile(r"(?:(?:\u8fd4\u56de|\u67e5|\u68c0\u7d22|\u53d6|\u524d)\s*)?(?P<count>\d{1,3})\s*(?:\u6761|\u6761\u6570|\u7bc7|\u4efd)")
@dataclass(frozen=True)
@@ -112,6 +112,86 @@ def _looks_like_url(value: str) -> bool:
return value.startswith(("http://", "https://"))
def _today() -> date:
return datetime.now(_BEIJING_TZ).date()
def _format_date(value: date) -> str:
return value.strftime("%Y-%m-%d")
def _subtract_months(value: date, months: int) -> date:
month_index = value.month - 1 - months
year = value.year + month_index // 12
month = month_index % 12 + 1
leap_year = year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
month_lengths = [31, 29 if leap_year else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
return value.replace(year=year, month=month, day=min(value.day, month_lengths[month - 1]))
def _normalize_date(value: str) -> str:
text = str(value or "").strip()
if not text:
return ""
match = _ISO_DATE_RE.search(text)
if not match:
return ""
try:
return _format_date(
date(
int(match.group("year")),
int(match.group("month")),
int(match.group("day")),
)
)
except ValueError:
return ""
def _extract_query_dates(query: str) -> tuple[str, str]:
dates = [_normalize_date(match.group(0)) for match in _ISO_DATE_RE.finditer(query)]
dates = [item for item in dates if item]
if len(dates) >= 2:
return min(dates[0], dates[1]), max(dates[0], dates[1])
if len(dates) == 1:
return dates[0], dates[0]
return "", ""
def _resolve_date_range(
query: str,
*,
start_date: str = "",
end_date: str = "",
today: date | None = None,
) -> tuple[str, str]:
normalized_start = _normalize_date(start_date)
normalized_end = _normalize_date(end_date)
if normalized_start or normalized_end:
return normalized_start, normalized_end
query_start, query_end = _extract_query_dates(query)
if query_start or query_end:
return query_start, query_end
current = today or _today()
if _RECENT_DAYS_RE.search(query):
return _format_date(current - timedelta(days=7)), _format_date(current)
if _RECENT_MONTHS_RE.search(query):
return _format_date(_subtract_months(current, 3)), _format_date(current)
return "", ""
def _resolve_top_k(query: str, top_k: str | int = "") -> str:
explicit = str(top_k).strip() if top_k is not None else ""
if explicit:
return explicit
match = _TOP_K_RE.search(query)
if not match:
return ""
return match.group("count")
def _error_result(query: str, message: str) -> str:
return json.dumps(
{
@@ -183,6 +263,10 @@ async def execute_search(
query: str,
settings: ConfigurableSearchSettings,
*,
start_date: str = "",
end_date: str = "",
top_k: str | int = "",
today: date | None = None,
transport: httpx.AsyncBaseTransport | None = None,
) -> str:
"""Execute a search request; ``transport`` exists for network-free tests."""
@@ -194,7 +278,16 @@ async def execute_search(
return _error_result(normalized_query, "web_search is disabled")
payload = deepcopy(settings.payload)
resolved_start, resolved_end = _resolve_date_range(
normalized_query,
start_date=start_date,
end_date=end_date,
today=today,
)
payload["query"] = normalized_query
payload["start_date"] = resolved_start
payload["end_date"] = resolved_end
payload["top_k"] = _resolve_top_k(normalized_query, top_k)
try:
async with httpx.AsyncClient(
@@ -234,19 +327,27 @@ async def execute_search(
@tool("web_search", parse_docstring=True)
async def web_search_tool(query: str) -> str:
"""Search the configured information service for current facts and source passages.
async def web_search_tool(query: str, start_date: str = "", end_date: str = "", top_k: str = "") -> str:
"""Search the internal knowledge base and return source passages.
Args:
query: Focused search keywords or a natural-language retrieval question.
Interpret relative time expressions against the current time in
Asia/Shanghai, and include concrete dates/date ranges in the query
when the user asks about recent periods.
query: Knowledge-base retrieval query. Keep the user's subject and time
requirement in this text. If the user says recent/近期/最近, this tool
will search the latest three months. If the user says near days/近几天/
最近几天/这几天/近一周, this tool will search the latest seven days.
start_date: Optional start date in YYYY-MM-DD. Leave empty by default.
Pass this only when the user explicitly asks for a concrete date or
date range.
end_date: Optional end date in YYYY-MM-DD. Leave empty by default.
Pass this only when the user explicitly asks for a concrete date or
date range.
top_k: Optional maximum number of records to return, such as "15".
Leave empty by default. Pass this when the user asks for a specific
number of results.
"""
try:
settings = ConfigurableSearchSettings.from_tool_config(get_app_config().get_tool_config("web_search"))
except ValueError as exc:
return _error_result(query, str(exc))
grounded_query = f"{query.strip()}\n当前时间: {_current_time_hint()}" if query.strip() else query
return await execute_search(grounded_query, settings)
return await execute_search(query, settings, start_date=start_date, end_date=end_date, top_k=top_k)
@@ -1,6 +1,7 @@
from __future__ import annotations
from __future__ import annotations
import json
from datetime import date
import httpx
import pytest
@@ -35,15 +36,20 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
async def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.method == "POST"
assert json.loads(request.content) == {"query": "测试检索词", "xxx": "xxxx"}
payload = json.loads(request.content)
assert payload["query"].strip()
assert payload["xxx"] == "xxxx"
assert payload["start_date"] == ""
assert payload["end_date"] == ""
assert payload["top_k"] == ""
return httpx.Response(
200,
json={
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"content": "test content",
"title": "test title",
"url": "https://original.example.test/one",
},
{
@@ -60,7 +66,7 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
settings = _settings()
result = json.loads(
await execute_search(
" 测试检索词 ",
" test query ",
settings,
transport=httpx.MockTransport(handler),
)
@@ -69,13 +75,13 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
assert len(requests) == 1
assert settings.payload == {"query": "configured value", "xxx": "xxxx"}
assert result == {
"query": "测试检索词",
"query": "test query",
"total_results": 2,
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"content": "test content",
"title": "test title",
"url": "https://search.example.test/mindId/xxx?recUuid=record%2F123",
},
{
@@ -104,6 +110,70 @@ async def test_configurable_search_handles_empty_results() -> None:
assert result == {"query": "nothing", "total_results": 0, "results": []}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("query", "expected_start", "expected_end"),
[
("近期政策", "2026-05-13", "2026-08-13"),
("近几天政策", "2026-08-06", "2026-08-13"),
("查询 2026年8月10日 到 2026-08-13 的政策", "2026-08-10", "2026-08-13"),
],
)
async def test_configurable_search_adds_date_payload(
query: str,
expected_start: str,
expected_end: str,
) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == expected_start
assert payload["end_date"] == expected_end
assert payload["top_k"] == ""
return httpx.Response(200, json={"results": []})
await execute_search(
query,
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_explicit_date_args_and_top_k_win() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == "2026-08-10"
assert payload["end_date"] == "2026-08-13"
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策",
_settings(),
start_date="2026/8/10",
end_date="2026年8月13日",
top_k=15,
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_infers_top_k_from_query() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策 返回15条",
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_accepts_root_array_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response: