feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
+23
-51
@@ -6,28 +6,25 @@ import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langgraph.typing import ContextT
|
||||
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
|
||||
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
|
||||
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
|
||||
|
||||
|
||||
_CONVERSATION_DEPOSIT_NAME = "对话沉淀"
|
||||
_CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
|
||||
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
|
||||
return (
|
||||
str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
|
||||
and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
|
||||
)
|
||||
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
|
||||
|
||||
|
||||
def _sanitize_conversation_deposit_value(value: Any) -> Any:
|
||||
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
|
||||
query: Focused natural-language retrieval query.
|
||||
"""
|
||||
|
||||
local_config = get_app_config().llmwiki.local_wiki_index
|
||||
resolved = get_resolved_llmwiki_runtime()
|
||||
if not resolved.weknora_enabled:
|
||||
return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
|
||||
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
|
||||
|
||||
store = get_llmwiki_store()
|
||||
user_id = get_effective_user_id()
|
||||
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
|
||||
return "No authorized LLMWiki knowledge bases are available."
|
||||
|
||||
try:
|
||||
results = await build_weknora_client(resolved).search(
|
||||
query.strip(),
|
||||
[str(row["weknora_id"]) for row in rows],
|
||||
)
|
||||
except (RuntimeError, WeKnoraError) as exc:
|
||||
return f"LLMWiki retrieval failed: {exc}"
|
||||
|
||||
local_by_remote = {str(row["weknora_id"]): row for row in rows}
|
||||
citations = []
|
||||
for result in results:
|
||||
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
visible_result = (
|
||||
_sanitize_conversation_deposit_value(result)
|
||||
if _is_conversation_deposit_mapping(mapping)
|
||||
else result
|
||||
)
|
||||
params = urlencode(
|
||||
{
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"chunk_id": visible_result.get("chunk_id") or "",
|
||||
"knowledge_id": visible_result.get("knowledge_id") or "",
|
||||
"document_id": visible_result.get("knowledge_id") or "",
|
||||
"wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
|
||||
}
|
||||
)
|
||||
citations.append(
|
||||
{
|
||||
"title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
|
||||
"content": visible_result.get("content") or "",
|
||||
"score": visible_result.get("score") or 0,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"knowledge_id": visible_result.get("knowledge_id") or "",
|
||||
"chunk_id": visible_result.get("chunk_id") or "",
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
}
|
||||
)
|
||||
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
|
||||
except (RuntimeError, WikiIndexError) as exc:
|
||||
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
|
||||
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
results = []
|
||||
for item in payload["results"]:
|
||||
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
|
||||
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
|
||||
results.append(visible)
|
||||
return json.dumps(
|
||||
{
|
||||
"query": query,
|
||||
"selection": [row["id"] for row in rows],
|
||||
"knowledge_base_count": len(rows),
|
||||
"results": citations,
|
||||
"mode": payload.get("mode") or "wiki_api",
|
||||
"results": results,
|
||||
"partial": payload.get("partial", False),
|
||||
"warnings": payload.get("warnings", []),
|
||||
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
Reference in new issue
Block a user