fix: improve llmwiki search integration
This commit is contained in:
1 parent
4809ee7db3
commit
56044d9cb7
20 files changed
+504
-66
No files matched your search
+64
-14
@@ -6,8 +6,10 @@ import json
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import httpx
|
||||
from langchain.tools import tool
|
||||
@@ -16,6 +18,12 @@ from deerflow.config import get_app_config
|
||||
from deerflow.config.tool_config import ToolConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_BEIJING_TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
def _current_time_hint() -> str:
|
||||
now = datetime.now(_BEIJING_TZ)
|
||||
return now.strftime("%Y-%m-%d %H:%M:%S, %A, Asia/Shanghai")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -87,6 +95,23 @@ def _text(value: Any) -> str:
|
||||
return str(value)
|
||||
|
||||
|
||||
def _metadata(item: dict[str, Any]) -> dict[str, Any]:
|
||||
value = item.get("metadata")
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _first_text(*values: Any) -> str:
|
||||
for value in values:
|
||||
text = _text(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _looks_like_url(value: str) -> bool:
|
||||
return value.startswith(("http://", "https://"))
|
||||
|
||||
|
||||
def _error_result(query: str, message: str) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
@@ -100,10 +125,13 @@ def _error_result(query: str, message: str) -> str:
|
||||
|
||||
|
||||
def _normalize_results(data: Any, settings: ConfigurableSearchSettings) -> list[dict[str, str]]:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("search API response must be a JSON object")
|
||||
if isinstance(data, list):
|
||||
raw_results = data
|
||||
elif isinstance(data, dict):
|
||||
raw_results = data.get("results")
|
||||
else:
|
||||
raise ValueError("search API response must be a JSON object or array")
|
||||
|
||||
raw_results = data.get("results")
|
||||
if not isinstance(raw_results, list):
|
||||
raise ValueError("search API response field 'results' must be an array")
|
||||
|
||||
@@ -113,8 +141,21 @@ def _normalize_results(data: Any, settings: ConfigurableSearchSettings) -> list[
|
||||
logger.warning("Ignoring non-object search result: %r", item)
|
||||
continue
|
||||
|
||||
rec_uuid = _text(item.get("recUuid")).strip()
|
||||
original_url = _text(item.get("url")).strip()
|
||||
metadata = _metadata(item)
|
||||
rec_uuid = _first_text(item.get("recUuid"), item.get("rec_uuid"), metadata.get("recUuid"), metadata.get("rec_uuid"))
|
||||
source = _first_text(item.get("source"), item.get("source1"), metadata.get("source"), metadata.get("source1"))
|
||||
original_url = _first_text(
|
||||
item.get("url"),
|
||||
item.get("link"),
|
||||
item.get("href"),
|
||||
item.get("source_url"),
|
||||
metadata.get("url"),
|
||||
metadata.get("link"),
|
||||
metadata.get("href"),
|
||||
metadata.get("source_url"),
|
||||
)
|
||||
if not original_url and _looks_like_url(source):
|
||||
original_url = source
|
||||
result_url = original_url
|
||||
if rec_uuid and settings.result_url_template:
|
||||
result_url = settings.result_url_template.replace(
|
||||
@@ -122,14 +163,19 @@ def _normalize_results(data: Any, settings: ConfigurableSearchSettings) -> list[
|
||||
quote(rec_uuid, safe=""),
|
||||
)
|
||||
|
||||
normalized.append(
|
||||
{
|
||||
"recUuid": rec_uuid,
|
||||
"content": _text(item.get("content")),
|
||||
"title": _text(item.get("title")),
|
||||
"url": result_url,
|
||||
}
|
||||
)
|
||||
normalized_item = {
|
||||
"recUuid": rec_uuid,
|
||||
"content": _first_text(item.get("content"), item.get("page_content"), metadata.get("content")),
|
||||
"title": _first_text(item.get("title"), item.get("m_title"), metadata.get("m_title")),
|
||||
"url": result_url,
|
||||
}
|
||||
if source:
|
||||
normalized_item["source"] = source
|
||||
publish_time = _first_text(item.get("time"), item.get("publish_time"), item.get("m_publish"), metadata.get("m_publish"))
|
||||
if publish_time:
|
||||
normalized_item["time"] = publish_time
|
||||
normalized_item["publish_time"] = publish_time
|
||||
normalized.append(normalized_item)
|
||||
return normalized
|
||||
|
||||
|
||||
@@ -193,10 +239,14 @@ async def web_search_tool(query: str) -> str:
|
||||
|
||||
Args:
|
||||
query: Focused search keywords or a natural-language retrieval question.
|
||||
Interpret relative time expressions against the current time in
|
||||
Asia/Shanghai, and include concrete dates/date ranges in the query
|
||||
when the user asks about recent periods.
|
||||
"""
|
||||
|
||||
try:
|
||||
settings = ConfigurableSearchSettings.from_tool_config(get_app_config().get_tool_config("web_search"))
|
||||
except ValueError as exc:
|
||||
return _error_result(query, str(exc))
|
||||
return await execute_search(query, settings)
|
||||
grounded_query = f"{query.strip()}\n当前时间: {_current_time_hint()}" if query.strip() else query
|
||||
return await execute_search(grounded_query, settings)
|
||||
Reference in new issue
Block a user