Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into dhk-dev
# Conflicts: # frontend-web/src/open-canvas/components/ai-writing/AIWritingForm.tsx # frontend-web/src/open-canvas/components/ai-writing/AIWritingPanel.tsx # offline-backend-20260512/backend/CLAUDE.md # offline-backend-20260512/backend/app/gateway/services.py # offline-backend-20260512/backend/tests/test_ai_writing_knowledge_base.py
This commit is contained in:
commit
221e947488
138 files changed
+65056
-2405
No files matched your search
+105
-12
@@ -127,6 +127,31 @@ def _fallback_knowledge_skill() -> SimpleNamespace:
|
||||
return SimpleNamespace(name=KNOWLEDGE_SEARCH_SKILL_NAME, description=_KNOWLEDGE_SKILL_FALLBACK_DESC)
|
||||
|
||||
|
||||
def _unique_limited(values: list, limit: int) -> list:
|
||||
"""Stable de-dup for short diagnostic lists."""
|
||||
return list(dict.fromkeys(values or []))[:limit]
|
||||
|
||||
|
||||
def _intranet_query_summary(rows: list[dict], limit: int = 5) -> str:
|
||||
"""Compact per-query stats for the visible skill step message."""
|
||||
parts: list[str] = []
|
||||
for row in rows[:limit]:
|
||||
query = str(row.get("query") or "")[:24] or "(空)"
|
||||
elapsed_ms = row.get("elapsed_ms")
|
||||
try:
|
||||
elapsed = f"{float(elapsed_ms) / 1000:.1f}s" if elapsed_ms is not None else ""
|
||||
except (TypeError, ValueError):
|
||||
elapsed = ""
|
||||
if row.get("error"):
|
||||
reason = str(row.get("error"))[:80]
|
||||
parts.append(f"{query}: 失败{f'/{elapsed}' if elapsed else ''}({reason})")
|
||||
else:
|
||||
parts.append(f"{query}: {int(row.get('count') or 0)} 条{f'/{elapsed}' if elapsed else ''}")
|
||||
if len(rows) > limit:
|
||||
parts.append(f"另 {len(rows) - limit} 个方向")
|
||||
return ";".join(parts)
|
||||
|
||||
|
||||
def _load_configured_skills() -> list:
|
||||
"""解析素材收集专家 agent(ai-writing-researcher)配置的技能列表。
|
||||
|
||||
@@ -370,7 +395,10 @@ async def _search_notebook(
|
||||
results: list[Material] = []
|
||||
mat_idx = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
# trust_env=False:笔记后端是内网/本机地址,但模型走公网,部署常设
|
||||
# HTTP_PROXY/HTTPS_PROXY;httpx 默认会把这个内网请求也走代理 → 不可达。
|
||||
# 强制忽略代理 env、直连(与内网 ES 检索同款修法)。
|
||||
async with httpx.AsyncClient(timeout=20, trust_env=False) as client:
|
||||
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
|
||||
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
|
||||
"query": kw,
|
||||
@@ -657,8 +685,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# 内网慢检索超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
||||
# 默认 600s,且最低 300s。
|
||||
timeout=_intranet_timeout_seconds(ai_writing_config),
|
||||
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
|
||||
)
|
||||
endpoint_label = intranet_url or getattr(provider, "endpoint", "内置默认内网端点")
|
||||
intranet_debug = {
|
||||
"mode": "knowledge_base",
|
||||
"configured": bool(intranet_url),
|
||||
"endpoint": endpoint_label,
|
||||
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
||||
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
||||
"verify_ssl": getattr(provider, "verify_ssl", False),
|
||||
"status": "running",
|
||||
"queries": [],
|
||||
"calls": [],
|
||||
"errors": [],
|
||||
"raw_responses": [],
|
||||
"result_count": 0,
|
||||
"used_as_fallback": False,
|
||||
}
|
||||
# 用户输入框追加(is_user_query_search)时 keywords 已是 [用户原话],全部检索;
|
||||
# 首次检索按 keyword_count 截断检索词。
|
||||
search_keywords = keywords if is_user_query_search else keywords[:keyword_count]
|
||||
@@ -670,20 +714,65 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
"message": f"使用「{_KNOWLEDGE_SKILL_DISPLAY}」直连内网接口({endpoint_label})检索 {total} 个方向:{('、'.join(search_keywords[:3]))[:60]}",
|
||||
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
|
||||
})
|
||||
|
||||
async def _search_one_keyword(kw: str) -> tuple[str, list[SearchResult], str, int]:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
results = await provider.search(kw, max_results=per_keyword)
|
||||
return kw, list(results or []), "", int((time.monotonic() - started) * 1000)
|
||||
except Exception as e: # noqa: BLE001 — provider 理论上会吞异常,这里兜住意外冒泡
|
||||
logger.warning("[knowledge-base] keyword=%r search failed: %s", kw, e)
|
||||
return kw, [], f"{type(e).__name__}: {e}", int((time.monotonic() - started) * 1000)
|
||||
|
||||
gathered = await asyncio.gather(
|
||||
*[provider.search(kw, max_results=per_keyword) for kw in search_keywords],
|
||||
*[_search_one_keyword(kw) for kw in search_keywords],
|
||||
return_exceptions=True,
|
||||
)
|
||||
for results in gathered:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("[knowledge-base] search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
for item in gathered:
|
||||
if isinstance(item, Exception):
|
||||
logger.warning("[knowledge-base] search task failed: %s", item)
|
||||
message = f"{type(item).__name__}: {item}"
|
||||
search_errors.append(message)
|
||||
intranet_debug["queries"].append({"query": "(unknown)", "count": 0, "error": message})
|
||||
continue
|
||||
kw, results, error, elapsed_ms = item
|
||||
row = {"query": kw, "count": len(results or []), "elapsed_ms": elapsed_ms}
|
||||
if error:
|
||||
row["error"] = error
|
||||
search_errors.append(error)
|
||||
intranet_debug["queries"].append(row)
|
||||
new_results.extend(results or [])
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
provider_errors = _unique_limited(getattr(provider, "errors", []) or [], 5)
|
||||
search_errors.extend(provider_errors)
|
||||
intranet_debug["errors"] = provider_errors
|
||||
intranet_debug["raw_responses"] = _unique_limited(getattr(provider, "raw_responses", []) or [], 3)
|
||||
intranet_debug["calls"] = list(getattr(provider, "calls", []) or [])[:20]
|
||||
intranet_debug["result_count"] = len(new_results)
|
||||
if search_errors and new_results:
|
||||
intranet_debug["status"] = "partial"
|
||||
elif search_errors:
|
||||
intranet_debug["status"] = "error"
|
||||
elif new_results:
|
||||
intranet_debug["status"] = "ok"
|
||||
else:
|
||||
intranet_debug["status"] = "empty"
|
||||
|
||||
query_summary = _intranet_query_summary(intranet_debug["queries"])
|
||||
if search_errors:
|
||||
uniq_errors = _unique_limited(search_errors, 3)
|
||||
_emit_skill_step({
|
||||
"type": "skill_error",
|
||||
"message": f"「{_KNOWLEDGE_SKILL_DISPLAY}」内网接口调用异常:{';'.join(uniq_errors)}",
|
||||
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
|
||||
"reason": ";".join(uniq_errors),
|
||||
})
|
||||
result_message = (
|
||||
f"「{_KNOWLEDGE_SKILL_DISPLAY}」(内网接口)调用完成,检索到 {len(new_results)} 条素材"
|
||||
+ (f"({query_summary})" if query_summary else "")
|
||||
)
|
||||
_emit_skill_step({
|
||||
"type": "skill_result",
|
||||
"message": f"「{_KNOWLEDGE_SKILL_DISPLAY}」(内网接口)检索到 {len(new_results)} 条素材",
|
||||
"message": result_message,
|
||||
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
|
||||
"matched_count": len(new_results),
|
||||
})
|
||||
@@ -758,7 +847,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
|
||||
)
|
||||
return (results or []), ""
|
||||
except asyncio.TimeoutError:
|
||||
except TimeoutError:
|
||||
msg = f"检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)"
|
||||
logger.warning("skill[%s] search timed out after %ss", skill_name, _SKILL_SEARCH_TIMEOUT_SECONDS)
|
||||
search_errors.append(f"skill[{skill_name}]: {msg}")
|
||||
@@ -901,16 +990,18 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
intranet_url = (ai_writing_config.get("intranet_search_url") or "").strip()
|
||||
# 诊断开关:开启后即使技能成功也照样跑完内网并记录完整结果(用于验证内网接口是否真能通)。
|
||||
intranet_debug_force = bool(ai_writing_config.get("intranet_search_debug", False))
|
||||
intranet_task: "asyncio.Task | None" = None
|
||||
intranet_task: asyncio.Task | None = None
|
||||
intranet_holder: dict = {}
|
||||
intranet_debug = {
|
||||
"configured": bool(intranet_url),
|
||||
"endpoint": intranet_url or "(未配置 ai_writing.intranet_search_url)",
|
||||
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
|
||||
"timeout": _intranet_timeout_seconds(ai_writing_config),
|
||||
"verify_ssl": bool(ai_writing_config.get("intranet_verify_ssl", False)),
|
||||
"debug_force": intranet_debug_force,
|
||||
"status": "pending" if intranet_url else "not_configured",
|
||||
"queries": [], # 每个检索词:{query, count, error?}
|
||||
"calls": [], # provider 每次真实 HTTP 调用,含简化词重试
|
||||
"errors": [], # provider 聚合的报错
|
||||
"raw_responses": [], # 解析失败时的原始响应片段
|
||||
"result_count": 0, # 内网总命中条数
|
||||
@@ -929,6 +1020,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# 兜底接口超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
|
||||
# 默认 600s(10 分钟),且最低 300s。
|
||||
timeout=_intranet_timeout_seconds(ai_writing_config),
|
||||
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
|
||||
)
|
||||
intranet_holder["provider"] = intranet
|
||||
fb_keywords = keywords[:keyword_count] or [intent]
|
||||
@@ -945,8 +1037,9 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
continue
|
||||
intranet_debug["queries"].append({"query": kw, "count": len(res or [])})
|
||||
collected.extend(res or [])
|
||||
intranet_debug["errors"] = list(dict.fromkeys(getattr(intranet, "errors", [])))[:5]
|
||||
intranet_debug["raw_responses"] = list(dict.fromkeys(getattr(intranet, "raw_responses", [])))[:3]
|
||||
intranet_debug["errors"] = _unique_limited(getattr(intranet, "errors", []) or [], 5)
|
||||
intranet_debug["raw_responses"] = _unique_limited(getattr(intranet, "raw_responses", []) or [], 3)
|
||||
intranet_debug["calls"] = list(getattr(intranet, "calls", []) or [])[:20]
|
||||
intranet_debug["result_count"] = len(collected)
|
||||
intranet_debug["status"] = "ran"
|
||||
return collected
|
||||
|
||||
+4
-2
@@ -1,8 +1,8 @@
|
||||
from .base import BaseSearchProvider, SearchResult
|
||||
from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout
|
||||
from .demo_search import DemoSearchProvider
|
||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||
from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
|
||||
from .skill_search import SkillSearchProvider
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
|
||||
intranet_knowledge_base: samples
|
||||
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
|
||||
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
|
||||
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
|
||||
|
||||
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
|
||||
@@ -26,6 +27,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
base_url=cfg.get("intranet_search_url", ""),
|
||||
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
|
||||
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
|
||||
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
|
||||
)
|
||||
if provider == "tavily":
|
||||
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
|
||||
|
||||
+133
-19
@@ -2,7 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
import time
|
||||
|
||||
from .base import BaseSearchProvider, SearchResult, simplify_query
|
||||
|
||||
@@ -14,6 +14,8 @@ _DEFAULT_KB = "samples"
|
||||
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
|
||||
_MIN_TIMEOUT_SECONDS = 300
|
||||
_DEFAULT_TIMEOUT_SECONDS = 600
|
||||
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
|
||||
_DEFAULT_VERIFY_SSL = False
|
||||
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
|
||||
_RAW_SNIPPET_LIMIT = 2000
|
||||
|
||||
@@ -27,6 +29,23 @@ def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_
|
||||
return max(value, _MIN_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
|
||||
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _pick(d: dict, keys: tuple) -> str:
|
||||
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
|
||||
for k in keys:
|
||||
@@ -38,6 +57,16 @@ def _pick(d: dict, keys: tuple) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _raw_snippet(body: str) -> str:
|
||||
"""Return a bounded raw response snippet for diagnostics."""
|
||||
body = (body or "").strip()
|
||||
if not body:
|
||||
return ""
|
||||
if len(body) > _RAW_SNIPPET_LIMIT:
|
||||
return body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
|
||||
return body
|
||||
|
||||
|
||||
class IntranetSearchProvider(BaseSearchProvider):
|
||||
"""内网知识库搜索接口适配器。
|
||||
|
||||
@@ -51,27 +80,29 @@ class IntranetSearchProvider(BaseSearchProvider):
|
||||
base_url: str = "",
|
||||
knowledge_base_name: str = _DEFAULT_KB,
|
||||
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
|
||||
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
|
||||
**_ignored,
|
||||
):
|
||||
self.endpoint = base_url or _DEFAULT_ENDPOINT
|
||||
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
|
||||
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
|
||||
self.timeout = normalize_intranet_timeout(timeout)
|
||||
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
|
||||
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
|
||||
self.errors: list[str] = []
|
||||
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
|
||||
self.raw_responses: list[str] = []
|
||||
# 每一次真实 HTTP 调用的诊断明细。search() 可能会因 0 结果触发简化词重试,
|
||||
# 因此一个业务关键词可能对应多条 calls。
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def _record_raw(self, body: str) -> None:
|
||||
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
|
||||
body = (body or "").strip()
|
||||
if not body:
|
||||
return
|
||||
if len(body) > _RAW_SNIPPET_LIMIT:
|
||||
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
|
||||
self.raw_responses.append(body)
|
||||
snippet = _raw_snippet(body)
|
||||
if snippet:
|
||||
self.raw_responses.append(snippet)
|
||||
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
|
||||
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
|
||||
results = await self._search_once(query, max_results)
|
||||
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
|
||||
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
|
||||
@@ -82,7 +113,7 @@ class IntranetSearchProvider(BaseSearchProvider):
|
||||
results = await self._search_once(simpler, max_results)
|
||||
return results
|
||||
|
||||
async def _search_once(self, query: str, max_results: int = 5) -> List[SearchResult]:
|
||||
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
@@ -92,44 +123,116 @@ class IntranetSearchProvider(BaseSearchProvider):
|
||||
}
|
||||
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
|
||||
raw_text = ""
|
||||
started = time.monotonic()
|
||||
call: dict = {
|
||||
"query": query,
|
||||
"endpoint": self.endpoint,
|
||||
"knowledge_base_name": self.knowledge_base_name,
|
||||
"max_results": max_results,
|
||||
"status": "pending",
|
||||
"ok": False,
|
||||
"count": 0,
|
||||
"verify_ssl": self.verify_ssl,
|
||||
}
|
||||
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
|
||||
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
|
||||
logger.info(
|
||||
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
|
||||
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
|
||||
# 服务器通常设了 HTTP_PROXY/HTTPS_PROXY 出公网;httpx 默认 trust_env=True
|
||||
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
|
||||
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
|
||||
# 调用同款修法)。
|
||||
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
|
||||
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
|
||||
resp = await client.post(self.endpoint, json=payload)
|
||||
call["http_status"] = getattr(resp, "status_code", None)
|
||||
resp.raise_for_status()
|
||||
raw_text = resp.text
|
||||
data = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
self._record_raw(e.response.text if e.response is not None else "")
|
||||
body_for_log = e.response.text if e.response is not None else ""
|
||||
logger.warning(
|
||||
"[intranet-search] HTTP error endpoint=%s query=%r status=%s body=%s",
|
||||
self.endpoint, query, getattr(e.response, "status_code", None), _raw_snippet(body_for_log),
|
||||
)
|
||||
message = f"{type(e).__name__}: {e}"
|
||||
self.errors.append(message)
|
||||
call.update({
|
||||
"status": "http_error",
|
||||
"error": message,
|
||||
"http_status": getattr(e.response, "status_code", None),
|
||||
})
|
||||
snippet = _raw_snippet(body_for_log)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self._record_raw(body_for_log)
|
||||
return []
|
||||
except Exception as e:
|
||||
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
|
||||
logger.warning("intranet search failed for query=%r: %s", query, e)
|
||||
self.errors.append(f"{type(e).__name__}: {e}")
|
||||
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体。
|
||||
# 打出异常类型,便于区分 ConnectError / ConnectTimeout / ReadTimeout /
|
||||
# RemoteProtocolError(空回复) / JSONDecodeError(返回非 JSON) 等具体根因。
|
||||
logger.warning(
|
||||
"[intranet-search] request failed endpoint=%s query=%r error=%s: %s",
|
||||
self.endpoint, query, type(e).__name__, e,
|
||||
)
|
||||
message = f"{type(e).__name__}: {e}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "error", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
finally:
|
||||
call["elapsed_ms"] = int((time.monotonic() - started) * 1000)
|
||||
# 非 pending 表示已经在异常分支写好;成功分支会在 return 前覆盖并追加。
|
||||
if call.get("status") != "pending":
|
||||
self.calls.append(call)
|
||||
|
||||
# 正常情况下接口直接返回数组;兼容包了一层 data/results 的结构(含工作正常的
|
||||
# knowledge-search-v2:{"success":true,"results":[...]})。
|
||||
if isinstance(data, list):
|
||||
items = data
|
||||
elif isinstance(data, dict):
|
||||
if "success" in data:
|
||||
call["response_success"] = bool(data.get("success"))
|
||||
if "count" in data:
|
||||
call["response_count"] = data.get("count")
|
||||
# 接口显式标失败时把整体回传前端便于定位
|
||||
if data.get("success") is False:
|
||||
self.errors.append(f"接口返回 success=false:{str(data.get('error') or '')[:200]}")
|
||||
message = f"接口返回 success=false:{str(data.get('error') or '')[:200]}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "success_false", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
items = data.get("data") or data.get("results") or data.get("documents") or []
|
||||
else:
|
||||
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
|
||||
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
|
||||
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
|
||||
message = f"返回结构异常:期望数组,实际为 {type(data).__name__}"
|
||||
self.errors.append(message)
|
||||
call.update({"status": "bad_shape", "error": message})
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
self._record_raw(raw_text)
|
||||
return []
|
||||
|
||||
results: List[SearchResult] = []
|
||||
logger.info(
|
||||
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
|
||||
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
|
||||
)
|
||||
results: list[SearchResult] = []
|
||||
for i, item in enumerate(items[:max_results]):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
@@ -158,6 +261,17 @@ class IntranetSearchProvider(BaseSearchProvider):
|
||||
source_type="intranet",
|
||||
recUuid=_pick(merged, ("recUuid", "rec_uuid", "id")),
|
||||
))
|
||||
call.update({
|
||||
"status": "ok" if results else "empty",
|
||||
"ok": True,
|
||||
"count": len(results),
|
||||
"item_count": len(items) if isinstance(items, list) else 0,
|
||||
})
|
||||
if not results:
|
||||
snippet = _raw_snippet(raw_text)
|
||||
if snippet:
|
||||
call["raw_response"] = snippet
|
||||
self.calls.append(call)
|
||||
return results
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
|
||||
@@ -40,6 +40,9 @@ class MaterialPackage(TypedDict, total=False):
|
||||
skill_search_used: bool
|
||||
# 技能检索的步骤记录(前端展示"使用了 X 技能"卡片)
|
||||
skill_search_steps: List[SkillStep]
|
||||
# 内网知识库接口诊断:端点、每个检索词的命中数/耗时/错误、原始响应片段等。
|
||||
# 用于排查 "检索很快返回 0 条" 时到底是否真正调用了内网接口。
|
||||
intranet_debug: dict
|
||||
|
||||
# ── 大纲 ──────────────────────────────────────────────────────────────────────
|
||||
# 三级结构:章节(##) → 小节(###) → 子要点(bullet -)。
|
||||
|
||||
+14
-9
@@ -12,9 +12,10 @@ from deerflow.agents.middlewares.prompt_prefix_middleware import PromptPrefixMid
|
||||
from deerflow.agents.middlewares.rag_citation_middleware import RagCitationMiddleware
|
||||
from deerflow.agents.middlewares.setup_agent_approval_middleware import SetupAgentApprovalMiddleware
|
||||
from deerflow.agents.middlewares.setup_writing_approval_middleware import SetupWritingApprovalMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
|
||||
from deerflow.agents.middlewares.summarization_middleware import BeforeSummarizationHook, DeerFlowSummarizationMiddleware
|
||||
from deerflow.agents.middlewares.task_external_context_middleware import TaskExternalContextMiddleware
|
||||
from deerflow.agents.middlewares.title_middleware import TitleMiddleware
|
||||
from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
|
||||
from deerflow.agents.middlewares.token_usage_middleware import TokenUsageMiddleware
|
||||
from deerflow.agents.middlewares.tool_error_handling_middleware import build_lead_runtime_middlewares
|
||||
@@ -326,12 +327,16 @@ def _build_middlewares(
|
||||
# to the global ``knowledge.rag_enabled`` config. No-op when knowledge is
|
||||
# disabled. Skipped in bootstrap/writing-setup/ephemeral like MemoryMiddleware.
|
||||
knowledge_cfg = getattr(resolved_app_config, "knowledge", None)
|
||||
if not is_bootstrap and not is_writing_setup and not is_ephemeral and knowledge_cfg is not None and knowledge_cfg.enabled:
|
||||
from deerflow.agents.middlewares.knowledge_rag_middleware import KnowledgeRagMiddleware
|
||||
|
||||
middlewares.append(KnowledgeRagMiddleware(limit=knowledge_cfg.rag_limit, default_enabled=knowledge_cfg.rag_enabled))
|
||||
|
||||
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
|
||||
if not is_bootstrap and not is_writing_setup and not is_ephemeral and knowledge_cfg is not None and knowledge_cfg.enabled:
|
||||
from deerflow.agents.middlewares.knowledge_rag_middleware import KnowledgeRagMiddleware
|
||||
|
||||
middlewares.append(KnowledgeRagMiddleware(limit=knowledge_cfg.rag_limit, default_enabled=knowledge_cfg.rag_enabled))
|
||||
|
||||
# TaskExternalContextMiddleware — inject hidden per-run business context
|
||||
# (for example rwfx's completed 3q report) into the model request only.
|
||||
middlewares.append(TaskExternalContextMiddleware())
|
||||
|
||||
# Add ViewImageMiddleware only for the NATIVE-vision path (main model
|
||||
# ingests raw image data). In delegated mode the view_image tool already
|
||||
# returns a text description and stores nothing in `viewed_images`, so the
|
||||
# middleware would otherwise inject a spurious "No images have been viewed."
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
"""Inject task-specific external context into the model request.
|
||||
|
||||
The web task workspace can pass business data through LangGraph run context
|
||||
without showing it in the chat transcript. This middleware converts that
|
||||
context into a transient message right before model invocation, so it is
|
||||
available to the agent but never persisted as user-visible conversation state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
_CONTEXT_KEY = "task_external_context"
|
||||
_TAG = "task_external_context"
|
||||
_MAX_CONTEXT_CHARS = 18000
|
||||
_TAIL_CONTEXT_CHARS = 2500
|
||||
|
||||
|
||||
def _truncate_middle(text: str, max_chars: int = _MAX_CONTEXT_CHARS) -> str:
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
marker = "\n\n...[隐藏任务上下文过长,已截断中间部分]...\n\n"
|
||||
tail_chars = min(_TAIL_CONTEXT_CHARS, max_chars // 4)
|
||||
head_chars = max(0, max_chars - len(marker) - tail_chars)
|
||||
return f"{text[:head_chars]}{marker}{text[-tail_chars:]}"
|
||||
|
||||
|
||||
def _render_context(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
try:
|
||||
text = json.dumps(value, ensure_ascii=False, indent=2, default=str).strip()
|
||||
except Exception:
|
||||
text = str(value).strip()
|
||||
return _truncate_middle(text)
|
||||
|
||||
|
||||
def _task_context_from_request(request: Any) -> Any:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
return context.get(_CONTEXT_KEY)
|
||||
return None
|
||||
|
||||
|
||||
def _build_context_message(value: Any) -> HumanMessage | None:
|
||||
text = _render_context(value)
|
||||
if not text:
|
||||
return None
|
||||
content = (
|
||||
f"<{_TAG}>\n"
|
||||
"[系统提示:以下是业务系统随本次运行提供的隐藏任务上下文,不是用户的新输入。"
|
||||
"回答当前任务时应作为背景资料参考;不要主动说明这些内容来自隐藏上下文,"
|
||||
"也不要在回答中暴露本标签本身。]\n\n"
|
||||
f"{text}\n"
|
||||
f"</{_TAG}>"
|
||||
)
|
||||
return HumanMessage(
|
||||
content=content,
|
||||
additional_kwargs={"hide_from_ui": True, _CONTEXT_KEY: True},
|
||||
)
|
||||
|
||||
|
||||
class TaskExternalContextMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Append hidden task context as a transient model-call message."""
|
||||
|
||||
state_schema = AgentState
|
||||
|
||||
@override
|
||||
async def awrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
msg = _build_context_message(_task_context_from_request(request))
|
||||
if msg is not None:
|
||||
request = request.override(messages=[*request.messages, msg])
|
||||
return await handler(request)
|
||||
|
||||
@override
|
||||
def wrap_model_call(self, request: Any, handler: Any) -> Any:
|
||||
msg = _build_context_message(_task_context_from_request(request))
|
||||
if msg is not None:
|
||||
request = request.override(messages=[*request.messages, msg])
|
||||
return handler(request)
|
||||
+159
-33
@@ -1,21 +1,49 @@
|
||||
"""Middleware for automatic thread title generation."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, NotRequired, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.config.title_config import get_title_config
|
||||
from deerflow.models import create_chat_model
|
||||
"""Middleware for automatic thread title generation."""
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, NotRequired, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from deerflow.config.title_config import get_title_config
|
||||
from deerflow.models import create_chat_model
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.title_config import TitleConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
_TITLE_DEBUG_LOGGER_NAME = "title_generation_debug"
|
||||
|
||||
|
||||
def _get_title_debug_logger() -> logging.Logger:
|
||||
debug_logger = logging.getLogger(_TITLE_DEBUG_LOGGER_NAME)
|
||||
if getattr(debug_logger, "_title_file_configured", False):
|
||||
return debug_logger
|
||||
|
||||
log_path = Path("logs/title_generation_debug.log")
|
||||
if not log_path.is_absolute():
|
||||
log_path = Path.cwd() / log_path
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
handler = logging.FileHandler(log_path, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
debug_logger.addHandler(handler)
|
||||
debug_logger.setLevel(logging.INFO)
|
||||
debug_logger.propagate = False
|
||||
setattr(debug_logger, "_title_file_configured", True)
|
||||
return debug_logger
|
||||
|
||||
|
||||
def log_title_debug(message: str, *args: Any) -> None:
|
||||
logger.warning(message, *args)
|
||||
_get_title_debug_logger().info(message, *args)
|
||||
|
||||
|
||||
class TitleMiddlewareState(AgentState):
|
||||
@@ -25,7 +53,7 @@ class TitleMiddlewareState(AgentState):
|
||||
title_provisional: NotRequired[bool]
|
||||
|
||||
|
||||
class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
"""Automatically generate a title for the thread after the first user message."""
|
||||
|
||||
state_schema = TitleMiddlewareState
|
||||
@@ -42,9 +70,9 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
return self._app_config.title
|
||||
return get_title_config()
|
||||
|
||||
def _normalize_content(self, content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
def _normalize_content(self, content: object) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
|
||||
if isinstance(content, list):
|
||||
parts = [self._normalize_content(item) for item in content]
|
||||
@@ -58,8 +86,18 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
nested_content = content.get("content")
|
||||
if nested_content is not None:
|
||||
return self._normalize_content(nested_content)
|
||||
|
||||
return ""
|
||||
|
||||
return ""
|
||||
|
||||
def _message_type(self, message: object) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("type") or message.get("role") or "").lower()
|
||||
return str(getattr(message, "type", "") or getattr(message, "role", "") or "").lower()
|
||||
|
||||
def _message_content(self, message: object) -> object:
|
||||
if isinstance(message, dict):
|
||||
return message.get("content", "")
|
||||
return getattr(message, "content", "")
|
||||
|
||||
def _should_set_provisional_title(self, state: TitleMiddlewareState) -> bool:
|
||||
"""Check if we should set an instant, question-derived placeholder title.
|
||||
@@ -78,8 +116,8 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
|
||||
# Only on the very first turn, as soon as the user message arrives.
|
||||
messages = state.get("messages", [])
|
||||
user_messages = [m for m in messages if m.type == "human"]
|
||||
return len(user_messages) == 1
|
||||
user_messages = [m for m in messages if self._message_type(m) in {"human", "user"}]
|
||||
return len(user_messages) == 1
|
||||
|
||||
def _should_generate_title(self, state: TitleMiddlewareState) -> bool:
|
||||
"""Check if we should generate the final LLM title for this thread."""
|
||||
@@ -98,8 +136,8 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
return False
|
||||
|
||||
# Count user and assistant messages
|
||||
user_messages = [m for m in messages if m.type == "human"]
|
||||
assistant_messages = [m for m in messages if m.type == "ai"]
|
||||
user_messages = [m for m in messages if self._message_type(m) in {"human", "user"}]
|
||||
assistant_messages = [m for m in messages if self._message_type(m) in {"ai", "assistant"}]
|
||||
|
||||
# Generate title after first complete exchange
|
||||
return len(user_messages) == 1 and len(assistant_messages) >= 1
|
||||
@@ -112,8 +150,8 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
config = self._get_title_config()
|
||||
messages = state.get("messages", [])
|
||||
|
||||
user_msg_content = next((m.content for m in messages if m.type == "human"), "")
|
||||
assistant_msg_content = next((m.content for m in messages if m.type == "ai"), "")
|
||||
user_msg_content = next((self._message_content(m) for m in messages if self._message_type(m) in {"human", "user"}), "")
|
||||
assistant_msg_content = next((self._message_content(m) for m in messages if self._message_type(m) in {"ai", "assistant"}), "")
|
||||
|
||||
user_msg = self._normalize_content(user_msg_content)
|
||||
assistant_msg = self._strip_think_tags(self._normalize_content(assistant_msg_content))
|
||||
@@ -174,7 +212,7 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
return None
|
||||
|
||||
messages = state.get("messages", [])
|
||||
user_msg_content = next((m.content for m in messages if m.type == "human"), "")
|
||||
user_msg_content = next((self._message_content(m) for m in messages if self._message_type(m) in {"human", "user"}), "")
|
||||
user_msg = self._normalize_content(user_msg_content)
|
||||
if not user_msg.strip():
|
||||
return None
|
||||
@@ -261,10 +299,98 @@ class TitleMiddleware(AgentMiddleware[TitleMiddlewareState]):
|
||||
async def abefore_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
return self._provisional_title_result(state)
|
||||
|
||||
@override
|
||||
def after_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
return self._generate_title_result(state)
|
||||
|
||||
@override
|
||||
async def aafter_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
return await self._agenerate_title_result(state)
|
||||
@override
|
||||
def after_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
# Keep the provisional title in place during the main run. The gateway
|
||||
# worker finalizes it in a background task after the answer is complete.
|
||||
return None
|
||||
|
||||
@override
|
||||
async def aafter_model(self, state: TitleMiddlewareState, runtime: Runtime) -> dict | None:
|
||||
# Keep the provisional title in place during the main async run so the
|
||||
# worker's post-run background finalizer can replace it with the real
|
||||
# LLM-generated title without blocking the input box.
|
||||
return None
|
||||
|
||||
|
||||
async def finalize_thread_title(
|
||||
*,
|
||||
checkpointer: Any,
|
||||
thread_store: Any | None,
|
||||
thread_id: str,
|
||||
source_state: dict[str, Any],
|
||||
app_config: "AppConfig | None" = None,
|
||||
) -> bool:
|
||||
"""Finalize a provisional thread title in the background.
|
||||
|
||||
``source_state`` must be the checkpoint snapshot captured when the first run
|
||||
completed. That avoids races where the user asks a second question before
|
||||
the background title task starts, which would otherwise make the current
|
||||
checkpoint fail the "first exchange only" predicate and leave the title
|
||||
provisional forever.
|
||||
"""
|
||||
|
||||
log_title_debug("Title finalize: evaluating background title generation for thread %s", thread_id)
|
||||
|
||||
middleware = TitleMiddleware(app_config=app_config)
|
||||
source_snapshot = copy.deepcopy(source_state)
|
||||
if not middleware._should_generate_title(source_snapshot):
|
||||
log_title_debug(
|
||||
"Title finalize: skipped for thread %s because source snapshot is not eligible",
|
||||
thread_id,
|
||||
)
|
||||
return False
|
||||
|
||||
log_title_debug("Title finalize: starting background title generation for thread %s", thread_id)
|
||||
result = await middleware._agenerate_title_result(source_snapshot)
|
||||
if not result:
|
||||
log_title_debug("Title finalize: no title result produced for thread %s", thread_id)
|
||||
return False
|
||||
|
||||
read_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(read_config)
|
||||
if checkpoint_tuple is None:
|
||||
log_title_debug("Title finalize: skipped for thread %s because checkpoint was not found", thread_id)
|
||||
return False
|
||||
|
||||
checkpoint: dict[str, Any] = dict(getattr(checkpoint_tuple, "checkpoint", {}) or {})
|
||||
metadata: dict[str, Any] = dict(getattr(checkpoint_tuple, "metadata", {}) or {})
|
||||
channel_values: dict[str, Any] = dict(checkpoint.get("channel_values", {}) or {})
|
||||
|
||||
# If another path already finalized or manually renamed the title, keep it.
|
||||
if channel_values.get("title_provisional") is not True:
|
||||
log_title_debug(
|
||||
"Title finalize: skipped for thread %s because title is no longer provisional",
|
||||
thread_id,
|
||||
)
|
||||
return False
|
||||
|
||||
channel_values.update(result)
|
||||
checkpoint["channel_values"] = channel_values
|
||||
metadata["updated_at"] = now_iso()
|
||||
|
||||
write_config = {
|
||||
"configurable": {
|
||||
"thread_id": thread_id,
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
new_versions: dict[str, Any] = {}
|
||||
get_next_version = getattr(checkpointer, "get_next_version", None)
|
||||
if callable(get_next_version):
|
||||
versions = dict(checkpoint.get("channel_versions", {}) or {})
|
||||
for key in ("title", "title_provisional"):
|
||||
next_version = get_next_version(versions.get(key), None)
|
||||
versions[key] = next_version
|
||||
new_versions[key] = next_version
|
||||
checkpoint["channel_versions"] = versions
|
||||
|
||||
await checkpointer.aput(write_config, checkpoint, metadata, new_versions)
|
||||
|
||||
title = result.get("title")
|
||||
if title and thread_store is not None:
|
||||
await thread_store.update_display_name(thread_id, title, user_id=None)
|
||||
|
||||
log_title_debug("Title finalize: completed for thread %s with title %r", thread_id, title)
|
||||
|
||||
return True
|
||||
Reference in new issue
Block a user