This commit is contained in:
董宏凯 committed 2026-08-14 11:15:37 +08:00
commit c4d244881f
14 files changed
+1194 -107

No files matched your search

@@ -1,12 +1,13 @@
"""Configurable JSON-over-HTTP implementation of the ``web_search`` tool."""
"""Configurable JSON-over-HTTP implementation of the ``web_search`` tool."""
from __future__ import annotations
import json
import logging
import re
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime
from datetime import date, datetime, timedelta
from typing import Any
from urllib.parse import quote
from zoneinfo import ZoneInfo
@@ -19,11 +20,10 @@ from deerflow.config.tool_config import ToolConfig
logger = logging.getLogger(__name__)
_BEIJING_TZ = ZoneInfo("Asia/Shanghai")
def _current_time_hint() -> str:
now = datetime.now(_BEIJING_TZ)
return now.strftime("%Y-%m-%d %H:%M:%S, %A, Asia/Shanghai")
_ISO_DATE_RE = re.compile(r"(?P<year>\d{4})(?:[-/.]|\u5e74)(?P<month>\d{1,2})(?:[-/.]|\u6708)(?P<day>\d{1,2})(?:\u65e5)?")
_RECENT_DAYS_RE = re.compile(r"(\u8fd1\u51e0\u5929|\u6700\u8fd1\u51e0\u5929|\u8fd9\u51e0\u5929|\u8fd1\u4e00\u5468|\u6700\u8fd1\u4e00\u5468|\u8fd17\u5929|\u6700\u8fd17\u5929)")
_RECENT_MONTHS_RE = re.compile(r"(\u8fd1\u671f|\u6700\u8fd1|\u8fd1\u4e09\u4e2a\u6708|\u6700\u8fd1\u4e09\u4e2a\u6708|\u8fd13\u4e2a\u6708|\u6700\u8fd13\u4e2a\u6708)")
_TOP_K_RE = re.compile(r"(?:(?:\u8fd4\u56de|\u67e5|\u68c0\u7d22|\u53d6|\u524d)\s*)?(?P<count>\d{1,3})\s*(?:\u6761|\u6761\u6570|\u7bc7|\u4efd)")
@dataclass(frozen=True)
@@ -112,6 +112,86 @@ def _looks_like_url(value: str) -> bool:
return value.startswith(("http://", "https://"))
def _today() -> date:
return datetime.now(_BEIJING_TZ).date()
def _format_date(value: date) -> str:
return value.strftime("%Y-%m-%d")
def _subtract_months(value: date, months: int) -> date:
month_index = value.month - 1 - months
year = value.year + month_index // 12
month = month_index % 12 + 1
leap_year = year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
month_lengths = [31, 29 if leap_year else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
return value.replace(year=year, month=month, day=min(value.day, month_lengths[month - 1]))
def _normalize_date(value: str) -> str:
text = str(value or "").strip()
if not text:
return ""
match = _ISO_DATE_RE.search(text)
if not match:
return ""
try:
return _format_date(
date(
int(match.group("year")),
int(match.group("month")),
int(match.group("day")),
)
)
except ValueError:
return ""
def _extract_query_dates(query: str) -> tuple[str, str]:
dates = [_normalize_date(match.group(0)) for match in _ISO_DATE_RE.finditer(query)]
dates = [item for item in dates if item]
if len(dates) >= 2:
return min(dates[0], dates[1]), max(dates[0], dates[1])
if len(dates) == 1:
return dates[0], dates[0]
return "", ""
def _resolve_date_range(
query: str,
*,
start_date: str = "",
end_date: str = "",
today: date | None = None,
) -> tuple[str, str]:
normalized_start = _normalize_date(start_date)
normalized_end = _normalize_date(end_date)
if normalized_start or normalized_end:
return normalized_start, normalized_end
query_start, query_end = _extract_query_dates(query)
if query_start or query_end:
return query_start, query_end
current = today or _today()
if _RECENT_DAYS_RE.search(query):
return _format_date(current - timedelta(days=7)), _format_date(current)
if _RECENT_MONTHS_RE.search(query):
return _format_date(_subtract_months(current, 3)), _format_date(current)
return "", ""
def _resolve_top_k(query: str, top_k: str | int = "") -> str:
explicit = str(top_k).strip() if top_k is not None else ""
if explicit:
return explicit
match = _TOP_K_RE.search(query)
if not match:
return ""
return match.group("count")
def _error_result(query: str, message: str) -> str:
return json.dumps(
{
@@ -183,6 +263,10 @@ async def execute_search(
query: str,
settings: ConfigurableSearchSettings,
*,
start_date: str = "",
end_date: str = "",
top_k: str | int = "",
today: date | None = None,
transport: httpx.AsyncBaseTransport | None = None,
) -> str:
"""Execute a search request; ``transport`` exists for network-free tests."""
@@ -194,7 +278,16 @@ async def execute_search(
return _error_result(normalized_query, "web_search is disabled")
payload = deepcopy(settings.payload)
resolved_start, resolved_end = _resolve_date_range(
normalized_query,
start_date=start_date,
end_date=end_date,
today=today,
)
payload["query"] = normalized_query
payload["start_date"] = resolved_start
payload["end_date"] = resolved_end
payload["top_k"] = _resolve_top_k(normalized_query, top_k)
try:
async with httpx.AsyncClient(
@@ -234,19 +327,27 @@ async def execute_search(
@tool("web_search", parse_docstring=True)
async def web_search_tool(query: str) -> str:
"""Search the configured information service for current facts and source passages.
async def web_search_tool(query: str, start_date: str = "", end_date: str = "", top_k: str = "") -> str:
"""Search the internal knowledge base and return source passages.
Args:
query: Focused search keywords or a natural-language retrieval question.
Interpret relative time expressions against the current time in
Asia/Shanghai, and include concrete dates/date ranges in the query
when the user asks about recent periods.
query: Knowledge-base retrieval query. Keep the user's subject and time
requirement in this text. If the user says recent/近期/最近, this tool
will search the latest three months. If the user says near days/近几天/
最近几天/这几天/近一周, this tool will search the latest seven days.
start_date: Optional start date in YYYY-MM-DD. Leave empty by default.
Pass this only when the user explicitly asks for a concrete date or
date range.
end_date: Optional end date in YYYY-MM-DD. Leave empty by default.
Pass this only when the user explicitly asks for a concrete date or
date range.
top_k: Optional maximum number of records to return, such as "15".
Leave empty by default. Pass this when the user asks for a specific
number of results.
"""
try:
settings = ConfigurableSearchSettings.from_tool_config(get_app_config().get_tool_config("web_search"))
except ValueError as exc:
return _error_result(query, str(exc))
grounded_query = f"{query.strip()}\n当前时间: {_current_time_hint()}" if query.strip() else query
return await execute_search(grounded_query, settings)
return await execute_search(query, settings, start_date=start_date, end_date=end_date, top_k=top_k)
@@ -38,16 +38,16 @@ class WeKnoraConnectionConfig(BaseModel):
admin_email: str = Field(
default="",
description=(
"Optional WeKnora administrator email used only by DeerFlow's "
"server-side detail-page iframe proxy. Prefer $ENV references."
"Optional WeKnora administrator email used by DeerFlow's server-side "
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
),
)
admin_password: str = Field(
default="",
repr=False,
description=(
"Optional WeKnora administrator password used only by DeerFlow's "
"server-side detail-page iframe proxy. Prefer $ENV references."
"Optional WeKnora administrator password used by DeerFlow's server-side "
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
),
)
model_config = ConfigDict(extra="ignore")
@@ -3,6 +3,7 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
import time
from typing import Any
from urllib.parse import quote
@@ -41,9 +42,10 @@ class WeKnoraClient:
"""Async client for a trusted-network WeKnora endpoint.
The caller may inject an ``httpx.AsyncClient`` for connection pooling and
tests. When omitted, each public method creates and closes a short-lived
client. DeerFlow deliberately sends no WeKnora credential; the independently
deployed service or its reverse proxy must allow DeerFlow's internal traffic.
tests. When omitted, each public method creates and closes a short-lived client.
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
mode for deployments that allow DeerFlow through a reverse proxy.
"""
def __init__(
@@ -52,13 +54,105 @@ class WeKnoraClient:
base_url: str,
timeout_seconds: float = 30.0,
http_client: httpx.AsyncClient | None = None,
admin_email: str = "",
admin_password: str = "",
) -> None:
self.base_url = base_url.rstrip("/")
self.timeout_seconds = timeout_seconds
self._http_client = http_client
self.admin_email = admin_email.strip()
self.admin_password = admin_password
self._auth_token = ""
self._tenant_id = ""
self._auth_expires_at = 0.0
if not self.base_url:
raise ValueError("WeKnora base URL is required")
@property
def _has_admin_credentials(self) -> bool:
return bool(self.admin_email and self.admin_password)
@staticmethod
def _extract_login_token(payload: Any) -> tuple[str, str]:
if not isinstance(payload, dict):
return "", ""
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
token = str(
source.get("token")
or source.get("access_token")
or payload.get("token")
or payload.get("access_token")
or ""
)
tenant = (
source.get("tenant")
if isinstance(source.get("tenant"), dict)
else source.get("active_tenant")
if isinstance(source.get("active_tenant"), dict)
else payload.get("tenant")
)
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
tenant_id = ""
if isinstance(tenant, dict):
tenant_id = str(tenant.get("id") or "")
if not tenant_id and isinstance(user, dict):
tenant_id = str(user.get("tenant_id") or "")
return token, tenant_id
async def _login(self, *, force_refresh: bool = False) -> None:
if not self._has_admin_credentials:
return
now = time.time()
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
)
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
self._auth_token = token
self._tenant_id = tenant_id
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
headers = {"Accept": accept}
if self._has_admin_credentials:
await self._login(force_refresh=force_refresh)
if self._auth_token:
headers["Authorization"] = f"Bearer {self._auth_token}"
if self._tenant_id:
headers["X-Tenant-ID"] = self._tenant_id
return headers
async def _request(
self,
method: str,
@@ -69,19 +163,21 @@ class WeKnoraClient:
files: Any = None,
data: Mapping[str, Any] | None = None,
) -> Any:
headers = {"Accept": "application/json"}
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout_seconds), follow_redirects=False)
try:
response = await client.request(
method,
f"{self.base_url}{path}",
headers=headers,
json=json,
params=params,
files=files,
data=data,
)
for attempt in range(2):
response = await client.request(
method,
f"{self.base_url}{path}",
headers=await self._headers(force_refresh=attempt > 0),
json=json,
params=params,
files=files,
data=data,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
@@ -92,6 +188,8 @@ class WeKnoraClient:
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
@@ -296,12 +394,15 @@ class WeKnoraClient:
follow_redirects=True,
)
try:
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge/{quote(knowledge_id, safe='')}/preview",
headers={"Accept": "*/*"},
follow_redirects=True,
)
for attempt in range(2):
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge/{quote(knowledge_id, safe='')}/preview",
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
follow_redirects=True,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora preview request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
@@ -312,6 +413,8 @@ class WeKnoraClient:
if response.status_code == 404:
raise WeKnoraError("WeKnora document preview was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated preview request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
@@ -40,4 +40,6 @@ def build_weknora_client(
return WeKnoraClient(
base_url=runtime.api_base_url,
http_client=http_client,
admin_email=runtime.admin_email,
admin_password=runtime.admin_password,
)
@@ -1,6 +1,7 @@
from __future__ import annotations
from __future__ import annotations
import json
from datetime import date
import httpx
import pytest
@@ -35,15 +36,20 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
async def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.method == "POST"
assert json.loads(request.content) == {"query": "测试检索词", "xxx": "xxxx"}
payload = json.loads(request.content)
assert payload["query"].strip()
assert payload["xxx"] == "xxxx"
assert payload["start_date"] == ""
assert payload["end_date"] == ""
assert payload["top_k"] == ""
return httpx.Response(
200,
json={
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"content": "test content",
"title": "test title",
"url": "https://original.example.test/one",
},
{
@@ -60,7 +66,7 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
settings = _settings()
result = json.loads(
await execute_search(
" 测试检索词 ",
" test query ",
settings,
transport=httpx.MockTransport(handler),
)
@@ -69,13 +75,13 @@ async def test_configurable_search_overrides_query_and_normalizes_nullable_resul
assert len(requests) == 1
assert settings.payload == {"query": "configured value", "xxx": "xxxx"}
assert result == {
"query": "测试检索词",
"query": "test query",
"total_results": 2,
"results": [
{
"recUuid": "record/123",
"content": "检索内容",
"title": "标题",
"content": "test content",
"title": "test title",
"url": "https://search.example.test/mindId/xxx?recUuid=record%2F123",
},
{
@@ -104,6 +110,70 @@ async def test_configurable_search_handles_empty_results() -> None:
assert result == {"query": "nothing", "total_results": 0, "results": []}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("query", "expected_start", "expected_end"),
[
("近期政策", "2026-05-13", "2026-08-13"),
("近几天政策", "2026-08-06", "2026-08-13"),
("查询 2026年8月10日 到 2026-08-13 的政策", "2026-08-10", "2026-08-13"),
],
)
async def test_configurable_search_adds_date_payload(
query: str,
expected_start: str,
expected_end: str,
) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == expected_start
assert payload["end_date"] == expected_end
assert payload["top_k"] == ""
return httpx.Response(200, json={"results": []})
await execute_search(
query,
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_explicit_date_args_and_top_k_win() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["start_date"] == "2026-08-10"
assert payload["end_date"] == "2026-08-13"
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策",
_settings(),
start_date="2026/8/10",
end_date="2026年8月13日",
top_k=15,
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_infers_top_k_from_query() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
payload = json.loads(request.content)
assert payload["top_k"] == "15"
return httpx.Response(200, json={"results": []})
await execute_search(
"近期政策 返回15条",
_settings(),
today=date(2026, 8, 13),
transport=httpx.MockTransport(handler),
)
@pytest.mark.asyncio
async def test_configurable_search_accepts_root_array_results() -> None:
async def handler(_: httpx.Request) -> httpx.Response: