feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -6,28 +6,25 @@ import json
import re
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlencode
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
_CONVERSATION_DEPOSIT_NAME = "对话沉淀"
_CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
return (
str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
)
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
def _sanitize_conversation_deposit_value(value: Any) -> Any:
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
query: Focused natural-language retrieval query.
"""
local_config = get_app_config().llmwiki.local_wiki_index
resolved = get_resolved_llmwiki_runtime()
if not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
store = get_llmwiki_store()
user_id = get_effective_user_id()
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
return "No authorized LLMWiki knowledge bases are available."
try:
results = await build_weknora_client(resolved).search(
query.strip(),
[str(row["weknora_id"]) for row in rows],
)
except (RuntimeError, WeKnoraError) as exc:
return f"LLMWiki retrieval failed: {exc}"
local_by_remote = {str(row["weknora_id"]): row for row in rows}
citations = []
for result in results:
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
visible_result = (
_sanitize_conversation_deposit_value(result)
if _is_conversation_deposit_mapping(mapping)
else result
)
params = urlencode(
{
"knowledge_base_id": mapping["id"],
"chunk_id": visible_result.get("chunk_id") or "",
"knowledge_id": visible_result.get("knowledge_id") or "",
"document_id": visible_result.get("knowledge_id") or "",
"wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
}
)
citations.append(
{
"title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
"content": visible_result.get("content") or "",
"score": visible_result.get("score") or 0,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"source": mapping.get("name") or "LLMWiki",
"knowledge_id": visible_result.get("knowledge_id") or "",
"chunk_id": visible_result.get("chunk_id") or "",
"url": f"/weknora-source-preview?{params}",
}
)
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
except (RuntimeError, WikiIndexError) as exc:
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
mapping_by_id = {str(row["id"]): row for row in rows}
results = []
for item in payload["results"]:
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
results.append(visible)
return json.dumps(
{
"query": query,
"selection": [row["id"] for row in rows],
"knowledge_base_count": len(rows),
"results": citations,
"mode": payload.get("mode") or "wiki_api",
"results": results,
"partial": payload.get("partial", False),
"warnings": payload.get("warnings", []),
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
},
ensure_ascii=False,
)