feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
@@ -26,8 +26,8 @@ from langchain_core.messages import HumanMessage
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
|
||||
from deerflow.persistence.llmwiki import LlmWikiStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
@@ -146,6 +146,24 @@ def _truncate(text: str, max_chars: int) -> str:
|
||||
|
||||
|
||||
def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
if result.get("kind") == "wiki_page":
|
||||
sections = list(result.get("matched_sections") or [])
|
||||
content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
|
||||
raw = {**result, "source": mapping.get("name") or "LLMWiki"}
|
||||
return {
|
||||
"index": index,
|
||||
"id": result.get("wiki_page_id") or result.get("id"),
|
||||
"title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
|
||||
"type": "llmwiki_wiki",
|
||||
"snippet": _truncate(content or str(result.get("summary") or ""), 360),
|
||||
"content": content,
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"score": float(result.get("score") or 0),
|
||||
"url": result.get("url") or "",
|
||||
"skillName": "llmwiki_search",
|
||||
"mode": "citation",
|
||||
"raw": raw,
|
||||
}
|
||||
chunk_id = str(result.get("chunk_id") or "")
|
||||
title = str(result.get("title") or result.get("filename") or mapping.get("name") or f"LLMWiki 片段 {index}")
|
||||
content = str(result.get("content") or "")
|
||||
@@ -175,10 +193,7 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
|
||||
|
||||
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
if not sources:
|
||||
return (
|
||||
"当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。"
|
||||
"请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
)
|
||||
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
|
||||
|
||||
lines = [
|
||||
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
|
||||
@@ -187,7 +202,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
"",
|
||||
f"用户问题:{query}",
|
||||
"",
|
||||
f"<llmwiki_reference_batch source_count=\"{len(sources)}\">",
|
||||
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
|
||||
]
|
||||
for source in sources:
|
||||
meta = " / ".join(
|
||||
@@ -203,7 +218,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
|
||||
lines.append(f"来源:{meta}")
|
||||
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
|
||||
lines.append("")
|
||||
lines.append("</llmwiki_reference_batch>")
|
||||
lines.append("</llmwiki_wiki_reference_batch>")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
@@ -218,7 +233,8 @@ async def build_llmwiki_rag_context(
|
||||
return LlmWikiRagContext([], 0, "", [], "")
|
||||
|
||||
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
|
||||
if not runtime.weknora_enabled:
|
||||
local_config = request.app.state.config.llmwiki.local_wiki_index
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
|
||||
return LlmWikiRagContext(selected_ids, 0, "", [], "")
|
||||
store = _store(request)
|
||||
if store is None:
|
||||
@@ -235,29 +251,25 @@ async def build_llmwiki_rag_context(
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
|
||||
rows.append(row)
|
||||
remote_to_local = {str(row["weknora_id"]): row for row in rows}
|
||||
|
||||
try:
|
||||
results = await build_weknora_client(runtime).search(query, [str(row["weknora_id"]) for row in rows])
|
||||
except WeKnoraError as exc:
|
||||
logger.warning(
|
||||
"LLMWiki pre-retrieval failed for user=%s kb_count=%s: %s",
|
||||
sanitize_log_param(user_id),
|
||||
len(rows),
|
||||
exc,
|
||||
)
|
||||
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
|
||||
if search_service is None:
|
||||
return LlmWikiRagContext(
|
||||
selected_ids,
|
||||
0,
|
||||
f"当前用户选择了 LLMWiki 知识库,但自动检索失败:{exc}。请说明暂时无法检索知识库背景资料,不要编造引用。",
|
||||
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
|
||||
[],
|
||||
query,
|
||||
)
|
||||
|
||||
try:
|
||||
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
|
||||
except WikiIndexError as exc:
|
||||
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
|
||||
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
sources: list[dict[str, Any]] = []
|
||||
for result in results:
|
||||
mapping = remote_to_local.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None or not str(result.get("content") or "").strip():
|
||||
for result in payload.get("results") or []:
|
||||
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
sources.append(_source_from_result(result, mapping, len(sources) + 1))
|
||||
if len(sources) >= _MAX_SOURCES:
|
||||
|
||||
Reference in new issue
Block a user