# Conflicts:
#	frontend-web/src/open-canvas/components/ai-writing/AIWritingForm.tsx
#	frontend-web/src/open-canvas/components/ai-writing/AIWritingPanel.tsx
#	offline-backend-20260512/backend/CLAUDE.md
#	offline-backend-20260512/backend/app/gateway/services.py
#	offline-backend-20260512/backend/tests/test_ai_writing_knowledge_base.py
This commit is contained in:
董宏凯 committed 2026-07-03 22:28:21 +08:00
commit 221e947488
138 files changed
+65056 -2405

No files matched your search

@@ -127,6 +127,31 @@ def _fallback_knowledge_skill() -> SimpleNamespace:
return SimpleNamespace(name=KNOWLEDGE_SEARCH_SKILL_NAME, description=_KNOWLEDGE_SKILL_FALLBACK_DESC)
def _unique_limited(values: list, limit: int) -> list:
"""Stable de-dup for short diagnostic lists."""
return list(dict.fromkeys(values or []))[:limit]
def _intranet_query_summary(rows: list[dict], limit: int = 5) -> str:
"""Compact per-query stats for the visible skill step message."""
parts: list[str] = []
for row in rows[:limit]:
query = str(row.get("query") or "")[:24] or "(空)"
elapsed_ms = row.get("elapsed_ms")
try:
elapsed = f"{float(elapsed_ms) / 1000:.1f}s" if elapsed_ms is not None else ""
except (TypeError, ValueError):
elapsed = ""
if row.get("error"):
reason = str(row.get("error"))[:80]
parts.append(f"{query}: 失败{f'/{elapsed}' if elapsed else ''}({reason})")
else:
parts.append(f"{query}: {int(row.get('count') or 0)} 条{f'/{elapsed}' if elapsed else ''}")
if len(rows) > limit:
parts.append(f"另 {len(rows) - limit} 个方向")
return ";".join(parts)
def _load_configured_skills() -> list:
"""解析素材收集专家 agent(ai-writing-researcher)配置的技能列表。
@@ -370,7 +395,10 @@ async def _search_notebook(
results: list[Material] = []
mat_idx = 0
try:
async with httpx.AsyncClient(timeout=20) as client:
# trust_env=False:笔记后端是内网/本机地址,但模型走公网,部署常设
# HTTP_PROXY/HTTPS_PROXY;httpx 默认会把这个内网请求也走代理 → 不可达。
# 强制忽略代理 env、直连(与内网 ES 检索同款修法)。
async with httpx.AsyncClient(timeout=20, trust_env=False) as client:
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
"query": kw,
@@ -657,8 +685,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 内网慢检索超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
# 默认 600s,且最低 300s。
timeout=_intranet_timeout_seconds(ai_writing_config),
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
)
endpoint_label = intranet_url or getattr(provider, "endpoint", "内置默认内网端点")
intranet_debug = {
"mode": "knowledge_base",
"configured": bool(intranet_url),
"endpoint": endpoint_label,
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
"timeout": _intranet_timeout_seconds(ai_writing_config),
"verify_ssl": getattr(provider, "verify_ssl", False),
"status": "running",
"queries": [],
"calls": [],
"errors": [],
"raw_responses": [],
"result_count": 0,
"used_as_fallback": False,
}
# 用户输入框追加(is_user_query_search)时 keywords 已是 [用户原话],全部检索;
# 首次检索按 keyword_count 截断检索词。
search_keywords = keywords if is_user_query_search else keywords[:keyword_count]
@@ -670,20 +714,65 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
"message": f"使用「{_KNOWLEDGE_SKILL_DISPLAY}」直连内网接口({endpoint_label})检索 {total} 个方向:{('、'.join(search_keywords[:3]))[:60]}",
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
})
async def _search_one_keyword(kw: str) -> tuple[str, list[SearchResult], str, int]:
started = time.monotonic()
try:
results = await provider.search(kw, max_results=per_keyword)
return kw, list(results or []), "", int((time.monotonic() - started) * 1000)
except Exception as e: # noqa: BLE001 — provider 理论上会吞异常,这里兜住意外冒泡
logger.warning("[knowledge-base] keyword=%r search failed: %s", kw, e)
return kw, [], f"{type(e).__name__}: {e}", int((time.monotonic() - started) * 1000)
gathered = await asyncio.gather(
*[provider.search(kw, max_results=per_keyword) for kw in search_keywords],
*[_search_one_keyword(kw) for kw in search_keywords],
return_exceptions=True,
)
for results in gathered:
if isinstance(results, Exception):
logger.warning("[knowledge-base] search failed: %s", results)
search_errors.append(f"{type(results).__name__}: {results}")
for item in gathered:
if isinstance(item, Exception):
logger.warning("[knowledge-base] search task failed: %s", item)
message = f"{type(item).__name__}: {item}"
search_errors.append(message)
intranet_debug["queries"].append({"query": "(unknown)", "count": 0, "error": message})
continue
kw, results, error, elapsed_ms = item
row = {"query": kw, "count": len(results or []), "elapsed_ms": elapsed_ms}
if error:
row["error"] = error
search_errors.append(error)
intranet_debug["queries"].append(row)
new_results.extend(results or [])
search_errors.extend(getattr(provider, "errors", []))
provider_errors = _unique_limited(getattr(provider, "errors", []) or [], 5)
search_errors.extend(provider_errors)
intranet_debug["errors"] = provider_errors
intranet_debug["raw_responses"] = _unique_limited(getattr(provider, "raw_responses", []) or [], 3)
intranet_debug["calls"] = list(getattr(provider, "calls", []) or [])[:20]
intranet_debug["result_count"] = len(new_results)
if search_errors and new_results:
intranet_debug["status"] = "partial"
elif search_errors:
intranet_debug["status"] = "error"
elif new_results:
intranet_debug["status"] = "ok"
else:
intranet_debug["status"] = "empty"
query_summary = _intranet_query_summary(intranet_debug["queries"])
if search_errors:
uniq_errors = _unique_limited(search_errors, 3)
_emit_skill_step({
"type": "skill_error",
"message": f"「{_KNOWLEDGE_SKILL_DISPLAY}」内网接口调用异常:{';'.join(uniq_errors)}",
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
"reason": ";".join(uniq_errors),
})
result_message = (
f"「{_KNOWLEDGE_SKILL_DISPLAY}」(内网接口)调用完成,检索到 {len(new_results)} 条素材"
+ (f"({query_summary})" if query_summary else "")
)
_emit_skill_step({
"type": "skill_result",
"message": f"「{_KNOWLEDGE_SKILL_DISPLAY}」(内网接口)检索到 {len(new_results)} 条素材",
"message": result_message,
"skill_name": KNOWLEDGE_SEARCH_SKILL_NAME,
"matched_count": len(new_results),
})
@@ -758,7 +847,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
timeout=_SKILL_SEARCH_TIMEOUT_SECONDS,
)
return (results or []), ""
except asyncio.TimeoutError:
except TimeoutError:
msg = f"检索超时(>{_SKILL_SEARCH_TIMEOUT_SECONDS}s)"
logger.warning("skill[%s] search timed out after %ss", skill_name, _SKILL_SEARCH_TIMEOUT_SECONDS)
search_errors.append(f"skill[{skill_name}]: {msg}")
@@ -901,16 +990,18 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
intranet_url = (ai_writing_config.get("intranet_search_url") or "").strip()
# 诊断开关:开启后即使技能成功也照样跑完内网并记录完整结果(用于验证内网接口是否真能通)。
intranet_debug_force = bool(ai_writing_config.get("intranet_search_debug", False))
intranet_task: "asyncio.Task | None" = None
intranet_task: asyncio.Task | None = None
intranet_holder: dict = {}
intranet_debug = {
"configured": bool(intranet_url),
"endpoint": intranet_url or "(未配置 ai_writing.intranet_search_url)",
"knowledge_base": ai_writing_config.get("intranet_knowledge_base", "samples"),
"timeout": _intranet_timeout_seconds(ai_writing_config),
"verify_ssl": bool(ai_writing_config.get("intranet_verify_ssl", False)),
"debug_force": intranet_debug_force,
"status": "pending" if intranet_url else "not_configured",
"queries": [], # 每个检索词:{query, count, error?}
"calls": [], # provider 每次真实 HTTP 调用,含简化词重试
"errors": [], # provider 聚合的报错
"raw_responses": [], # 解析失败时的原始响应片段
"result_count": 0, # 内网总命中条数
@@ -929,6 +1020,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 兜底接口超时可在 config.yaml 的 ai_writing.intranet_search_timeout 调大,
# 默认 600s(10 分钟),且最低 300s。
timeout=_intranet_timeout_seconds(ai_writing_config),
verify_ssl=ai_writing_config.get("intranet_verify_ssl", False),
)
intranet_holder["provider"] = intranet
fb_keywords = keywords[:keyword_count] or [intent]
@@ -945,8 +1037,9 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
continue
intranet_debug["queries"].append({"query": kw, "count": len(res or [])})
collected.extend(res or [])
intranet_debug["errors"] = list(dict.fromkeys(getattr(intranet, "errors", [])))[:5]
intranet_debug["raw_responses"] = list(dict.fromkeys(getattr(intranet, "raw_responses", [])))[:3]
intranet_debug["errors"] = _unique_limited(getattr(intranet, "errors", []) or [], 5)
intranet_debug["raw_responses"] = _unique_limited(getattr(intranet, "raw_responses", []) or [], 3)
intranet_debug["calls"] = list(getattr(intranet, "calls", []) or [])[:20]
intranet_debug["result_count"] = len(collected)
intranet_debug["status"] = "ran"
return collected
@@ -1,8 +1,8 @@
from .base import BaseSearchProvider, SearchResult
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
from .skill_search import SkillSearchProvider
@@ -14,6 +14,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
intranet_knowledge_base: samples
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
@@ -26,6 +27,7 @@ def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
base_url=cfg.get("intranet_search_url", ""),
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
)
if provider == "tavily":
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
@@ -2,7 +2,7 @@
from __future__ import annotations
import logging
from typing import List
import time
from .base import BaseSearchProvider, SearchResult, simplify_query
@@ -14,6 +14,8 @@ _DEFAULT_KB = "samples"
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
_MIN_TIMEOUT_SECONDS = 300
_DEFAULT_TIMEOUT_SECONDS = 600
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
_DEFAULT_VERIFY_SSL = False
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
_RAW_SNIPPET_LIMIT = 2000
@@ -27,6 +29,23 @@ def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_
return max(value, _MIN_TIMEOUT_SECONDS)
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
if isinstance(value, bool):
return value
if value is None:
return default
if isinstance(value, (int, float)):
return bool(value)
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return default
def _pick(d: dict, keys: tuple) -> str:
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
for k in keys:
@@ -38,6 +57,16 @@ def _pick(d: dict, keys: tuple) -> str:
return ""
def _raw_snippet(body: str) -> str:
"""Return a bounded raw response snippet for diagnostics."""
body = (body or "").strip()
if not body:
return ""
if len(body) > _RAW_SNIPPET_LIMIT:
return body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
return body
class IntranetSearchProvider(BaseSearchProvider):
"""内网知识库搜索接口适配器。
@@ -51,27 +80,29 @@ class IntranetSearchProvider(BaseSearchProvider):
base_url: str = "",
knowledge_base_name: str = _DEFAULT_KB,
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
**_ignored,
):
self.endpoint = base_url or _DEFAULT_ENDPOINT
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
self.timeout = normalize_intranet_timeout(timeout)
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
self.errors: list[str] = []
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
self.raw_responses: list[str] = []
# 每一次真实 HTTP 调用的诊断明细。search() 可能会因 0 结果触发简化词重试,
# 因此一个业务关键词可能对应多条 calls。
self.calls: list[dict] = []
def _record_raw(self, body: str) -> None:
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
body = (body or "").strip()
if not body:
return
if len(body) > _RAW_SNIPPET_LIMIT:
body = body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
self.raw_responses.append(body)
snippet = _raw_snippet(body)
if snippet:
self.raw_responses.append(snippet)
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
results = await self._search_once(query, max_results)
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
@@ -82,7 +113,7 @@ class IntranetSearchProvider(BaseSearchProvider):
results = await self._search_once(simpler, max_results)
return results
async def _search_once(self, query: str, max_results: int = 5) -> List[SearchResult]:
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
import httpx
payload = {
@@ -92,44 +123,116 @@ class IntranetSearchProvider(BaseSearchProvider):
}
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
raw_text = ""
started = time.monotonic()
call: dict = {
"query": query,
"endpoint": self.endpoint,
"knowledge_base_name": self.knowledge_base_name,
"max_results": max_results,
"status": "pending",
"ok": False,
"count": 0,
"verify_ssl": self.verify_ssl,
}
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
logger.info(
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
)
try:
async with httpx.AsyncClient(timeout=self.timeout) as client:
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
# 服务器通常设了 HTTP_PROXY/HTTPS_PROXY 出公网;httpx 默认 trust_env=True
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
# 调用同款修法)。
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
resp = await client.post(self.endpoint, json=payload)
call["http_status"] = getattr(resp, "status_code", None)
resp.raise_for_status()
raw_text = resp.text
data = resp.json()
except httpx.HTTPStatusError as e:
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
self._record_raw(e.response.text if e.response is not None else "")
body_for_log = e.response.text if e.response is not None else ""
logger.warning(
"[intranet-search] HTTP error endpoint=%s query=%r status=%s body=%s",
self.endpoint, query, getattr(e.response, "status_code", None), _raw_snippet(body_for_log),
)
message = f"{type(e).__name__}: {e}"
self.errors.append(message)
call.update({
"status": "http_error",
"error": message,
"http_status": getattr(e.response, "status_code", None),
})
snippet = _raw_snippet(body_for_log)
if snippet:
call["raw_response"] = snippet
self._record_raw(body_for_log)
return []
except Exception as e:
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体
logger.warning("intranet search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体。
# 打出异常类型,便于区分 ConnectError / ConnectTimeout / ReadTimeout /
# RemoteProtocolError(空回复) / JSONDecodeError(返回非 JSON) 等具体根因。
logger.warning(
"[intranet-search] request failed endpoint=%s query=%r error=%s: %s",
self.endpoint, query, type(e).__name__, e,
)
message = f"{type(e).__name__}: {e}"
self.errors.append(message)
call.update({"status": "error", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self._record_raw(raw_text)
return []
finally:
call["elapsed_ms"] = int((time.monotonic() - started) * 1000)
# 非 pending 表示已经在异常分支写好;成功分支会在 return 前覆盖并追加。
if call.get("status") != "pending":
self.calls.append(call)
# 正常情况下接口直接返回数组;兼容包了一层 data/results 的结构(含工作正常的
# knowledge-search-v2:{"success":true,"results":[...]})。
if isinstance(data, list):
items = data
elif isinstance(data, dict):
if "success" in data:
call["response_success"] = bool(data.get("success"))
if "count" in data:
call["response_count"] = data.get("count")
# 接口显式标失败时把整体回传前端便于定位
if data.get("success") is False:
self.errors.append(f"接口返回 success=false:{str(data.get('error') or '')[:200]}")
message = f"接口返回 success=false:{str(data.get('error') or '')[:200]}"
self.errors.append(message)
call.update({"status": "success_false", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
self._record_raw(raw_text)
return []
items = data.get("data") or data.get("results") or data.get("documents") or []
else:
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
self.errors.append(f"返回结构异常:期望数组,实际为 {type(data).__name__}")
message = f"返回结构异常:期望数组,实际为 {type(data).__name__}"
self.errors.append(message)
call.update({"status": "bad_shape", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
self._record_raw(raw_text)
return []
results: List[SearchResult] = []
logger.info(
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
)
results: list[SearchResult] = []
for i, item in enumerate(items[:max_results]):
if not isinstance(item, dict):
continue
@@ -158,6 +261,17 @@ class IntranetSearchProvider(BaseSearchProvider):
source_type="intranet",
recUuid=_pick(merged, ("recUuid", "rec_uuid", "id")),
))
call.update({
"status": "ok" if results else "empty",
"ok": True,
"count": len(results),
"item_count": len(items) if isinstance(items, list) else 0,
})
if not results:
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
return results
async def health_check(self) -> bool:
@@ -40,6 +40,9 @@ class MaterialPackage(TypedDict, total=False):
skill_search_used: bool
# 技能检索的步骤记录(前端展示"使用了 X 技能"卡片)
skill_search_steps: List[SkillStep]
# 内网知识库接口诊断:端点、每个检索词的命中数/耗时/错误、原始响应片段等。
# 用于排查 "检索很快返回 0 条" 时到底是否真正调用了内网接口。
intranet_debug: dict
# ── 大纲 ──────────────────────────────────────────────────────────────────────
# 三级结构:章节(##) → 小节(###) → 子要点(bullet -)。