初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,5 @@
"""WeKnora REST integration."""
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
__all__ = ["WeKnoraClient", "WeKnoraError"]
@@ -0,0 +1,674 @@
"""Small, version-pinned adapter over WeKnora's official REST API."""
from __future__ import annotations
import time
from collections.abc import Mapping, Sequence
from typing import Any
from urllib.parse import quote
import httpx
class WeKnoraError(RuntimeError):
"""Normalized upstream failure that is safe for DeerFlow to translate."""
def __init__(self, message: str, *, status_code: int = 502) -> None:
super().__init__(message)
self.status_code = status_code
def _data(payload: Any) -> Any:
if not isinstance(payload, dict):
raise WeKnoraError("WeKnora returned an invalid response")
if payload.get("success") is False:
message = payload.get("message") or payload.get("error") or "WeKnora rejected the request"
raise WeKnoraError(str(message), status_code=400)
return payload.get("data", payload)
def _as_list(value: Any) -> list[dict[str, Any]]:
if isinstance(value, list):
return [dict(item) for item in value if isinstance(item, Mapping)]
if isinstance(value, dict):
for key in ("items", "list", "knowledge_bases", "knowledge", "records"):
nested = value.get(key)
if isinstance(nested, list):
return [dict(item) for item in nested if isinstance(item, Mapping)]
return []
class WeKnoraClient:
"""Async client for a trusted-network WeKnora endpoint.
The caller may inject an ``httpx.AsyncClient`` for connection pooling and
tests. When omitted, each public method creates and closes a short-lived client.
If admin credentials are configured, DeerFlow authenticates to WeKnora with a
cached bearer token; otherwise it keeps the legacy trusted-network anonymous
mode for deployments that allow DeerFlow through a reverse proxy.
"""
def __init__(
self,
*,
base_url: str,
timeout_seconds: float = 30.0,
http_client: httpx.AsyncClient | None = None,
admin_email: str = "",
admin_password: str = "",
) -> None:
self.base_url = base_url.rstrip("/")
self.timeout_seconds = timeout_seconds
self._http_client = http_client
self.admin_email = admin_email.strip()
self.admin_password = admin_password
self._auth_token = ""
self._tenant_id = ""
self._auth_expires_at = 0.0
if not self.base_url:
raise ValueError("WeKnora base URL is required")
@property
def _has_admin_credentials(self) -> bool:
return bool(self.admin_email and self.admin_password)
@staticmethod
def _extract_login_token(payload: Any) -> tuple[str, str]:
if not isinstance(payload, dict):
return "", ""
source = payload.get("data") if isinstance(payload.get("data"), dict) else payload
token = str(
source.get("token")
or source.get("access_token")
or payload.get("token")
or payload.get("access_token")
or ""
)
tenant = (
source.get("tenant")
if isinstance(source.get("tenant"), dict)
else source.get("active_tenant")
if isinstance(source.get("active_tenant"), dict)
else payload.get("tenant")
)
user = source.get("user") if isinstance(source.get("user"), dict) else payload.get("user")
tenant_id = ""
if isinstance(tenant, dict):
tenant_id = str(tenant.get("id") or "")
if not tenant_id and isinstance(user, dict):
tenant_id = str(user.get("tenant_id") or "")
return token, tenant_id
async def _login(self, *, force_refresh: bool = False) -> None:
if not self._has_admin_credentials:
return
now = time.time()
if self._auth_token and not force_refresh and self._auth_expires_at > now + 60:
return
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
try:
response = await client.post(
f"{self.base_url}/api/v1/auth/login",
json={"email": self.admin_email, "password": self.admin_password},
headers={"Accept": "application/json"},
)
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora admin login timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora admin login is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora admin login failed ({response.status_code})", status_code=502)
try:
payload = response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora admin login returned invalid JSON", status_code=502) from exc
if isinstance(payload, dict) and payload.get("success") is False:
raise WeKnoraError(str(payload.get("message") or "WeKnora admin login failed"), status_code=502)
token, tenant_id = self._extract_login_token(payload)
if not token:
raise WeKnoraError("WeKnora admin login did not return a token", status_code=502)
self._auth_token = token
self._tenant_id = tenant_id
# WeKnora login responses do not consistently expose expiry. Refresh
# periodically and retry once if upstream says the token is stale.
self._auth_expires_at = now + 25 * 60
async def _headers(self, *, force_refresh: bool = False, accept: str = "application/json") -> dict[str, str]:
headers = {"Accept": accept}
if self._has_admin_credentials:
await self._login(force_refresh=force_refresh)
if self._auth_token:
headers["Authorization"] = f"Bearer {self._auth_token}"
if self._tenant_id:
headers["X-Tenant-ID"] = self._tenant_id
return headers
async def _request(
self,
method: str,
path: str,
*,
json: Any = None,
params: Mapping[str, Any] | None = None,
files: Any = None,
data: Mapping[str, Any] | None = None,
) -> Any:
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=False,
trust_env=False,
)
try:
for attempt in range(2):
response = await client.request(
method,
f"{self.base_url}{path}",
headers=await self._headers(force_refresh=attempt > 0),
json=json,
params=params,
files=files,
data=data,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora resource was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated internal request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
detail = ""
try:
body = response.json()
if isinstance(body, dict):
detail = str(body.get("message") or body.get("error") or body.get("detail") or "")
except ValueError:
detail = ""
raise WeKnoraError(detail or f"WeKnora returned HTTP {response.status_code}", status_code=502)
if response.status_code == 204 or not response.content:
return {"success": True}
try:
return response.json()
except ValueError as exc:
raise WeKnoraError("WeKnora returned invalid JSON") from exc
async def health(self) -> dict[str, Any]:
payload = await self._request("GET", "/health")
return dict(payload) if isinstance(payload, dict) else {"status": "ok"}
async def list_knowledge_bases(self) -> list[dict[str, Any]]:
return _as_list(_data(await self._request("GET", "/api/v1/knowledge-bases")))
async def get_knowledge_base(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid knowledge base")
return dict(value)
async def list_models(self) -> list[dict[str, Any]]:
"""List models from WeKnora; model credentials are never returned."""
return _as_list(_data(await self._request("GET", "/api/v1/models")))
async def get_default_embedding_model_id(self) -> str:
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "embedding"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
raise WeKnoraError(
"WeKnora has no available embedding model; configure its default model in WeKnora",
status_code=400,
)
return model_id
async def get_default_knowledge_qa_model_id(self) -> str:
"""Resolve WeKnora's own active synthesis model for Wiki indexing."""
models = [item for item in await self.list_models() if str(item.get("type") or "").strip().lower() == "knowledgeqa"]
preferred = next((item for item in models if item.get("is_default") is True), None)
selected = preferred or next(
(item for item in models if str(item.get("status") or "active").lower() == "active"),
None,
)
model_id = str((selected or {}).get("id") or "")
if not model_id:
raise WeKnoraError(
"WeKnora has no available KnowledgeQA model; configure one in WeKnora before creating a Wiki",
status_code=400,
)
return model_id
async def create_knowledge_base(
self,
*,
name: str,
description: str = "",
kb_type: str = "document",
embedding_model_id: str | None = None,
wiki_enabled: bool = False,
) -> dict[str, Any]:
model_id = embedding_model_id or await self.get_default_embedding_model_id()
create_payload: dict[str, Any] = {
"name": name,
"description": description,
"type": kb_type,
"embedding_model_id": model_id,
}
if wiki_enabled:
synthesis_model_id = await self.get_default_knowledge_qa_model_id()
create_payload.update(
{
"summary_model_id": synthesis_model_id,
"indexing_strategy": {
"vector_enabled": True,
"keyword_enabled": True,
"wiki_enabled": True,
"graph_enabled": False,
},
"wiki_config": {
"synthesis_model_id": synthesis_model_id,
"extraction_granularity": "standard",
},
}
)
payload = await self._request(
"POST",
"/api/v1/knowledge-bases",
json=create_payload,
)
value = _data(payload)
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("WeKnora did not return the created knowledge base")
return dict(value)
async def update_knowledge_base(self, knowledge_base_id: str, changes: Mapping[str, Any]) -> dict[str, Any]:
payload = dict(changes)
if not str(payload.get("name") or "").strip():
current = await self.get_knowledge_base(knowledge_base_id)
current_name = str(current.get("name") or "").strip()
if not current_name:
raise WeKnoraError("WeKnora knowledge base name is required for updates")
payload["name"] = current_name
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}",
json=payload,
)
)
return dict(value) if isinstance(value, dict) else {}
async def delete_knowledge_base(self, knowledge_base_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}")
async def list_documents(self, knowledge_base_id: str, *, page: int = 1, page_size: int = 100) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge",
params={"page": page, "page_size": page_size},
)
rows = _as_list(_data(payload))
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
async def upload_document(
self,
knowledge_base_id: str,
*,
filename: str,
content: bytes,
content_type: str = "application/octet-stream",
) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/file",
files={"file": (filename, content, content_type)},
)
)
return dict(value) if isinstance(value, dict) else {}
async def import_document_url(self, knowledge_base_id: str, *, url: str) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/url",
json={"url": url},
)
)
return dict(value) if isinstance(value, dict) else {}
async def create_manual_document(
self,
knowledge_base_id: str,
*,
title: str,
content: str,
) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/knowledge/manual",
json={"title": title, "content": content, "status": "publish"},
)
)
return dict(value) if isinstance(value, dict) else {}
async def delete_document(self, knowledge_id: str) -> None:
await self._request("DELETE", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}")
async def get_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid document")
return dict(value)
async def get_document_preview(self, knowledge_id: str) -> tuple[bytes, str]:
"""Return the original document preview bytes and its media type."""
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=True,
trust_env=False,
)
try:
for attempt in range(2):
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge/{quote(knowledge_id, safe='')}/preview",
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
follow_redirects=True,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora preview request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora preview is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora document preview was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated preview request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def get_knowledge_base_file(
self,
knowledge_base_id: str,
file_path: str,
) -> tuple[bytes, str]:
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=True,
trust_env=False,
)
try:
for attempt in range(2):
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
params={"file_path": file_path},
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
follow_redirects=True,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
return dict(value) if isinstance(value, dict) else {}
async def list_chunks(self, knowledge_id: str, *, page: int = 1, page_size: int = 20) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/chunks/{quote(knowledge_id, safe='')}",
params={"page": page, "page_size": page_size},
)
rows = _as_list(_data(payload))
total = payload.get("total", len(rows)) if isinstance(payload, dict) else len(rows)
return {"items": rows, "total": int(total or 0), "page": page, "page_size": page_size}
async def get_wiki_graph(self, knowledge_base_id: str, *, limit: int = 200) -> dict[str, Any]:
value = _data(
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki generation status")
return value
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("Invalid Wiki folder response")
return value
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
async def list_wiki_pages(
self,
knowledge_base_id: str,
*,
page: int = 1,
page_size: int = 50,
query: str = "",
) -> dict[str, Any]:
payload = await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
params={"page": page, "page_size": page_size, "query": query or None},
)
value = _data(payload)
source = value if isinstance(value, dict) else payload if isinstance(payload, dict) else {}
rows = _as_list(source.get("pages") if isinstance(source, dict) else value)
return {
"pages": rows,
"total": int(source.get("total") or len(rows)),
"page": int(source.get("page") or page),
"page_size": int(source.get("page_size") or page_size),
"total_pages": int(source.get("total_pages") or max(1, (int(source.get("total") or len(rows)) + page_size - 1) // page_size)),
}
async def get_wiki_page(self, knowledge_base_id: str, slug: str) -> dict[str, Any]:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
value = _data(
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid Wiki page")
return dict(value)
async def get_wiki_index(
self,
knowledge_base_id: str,
*,
types: Sequence[str],
limit: int = 50,
cursor: str = "",
) -> dict[str, Any]:
params: dict[str, Any] = {
"types": ",".join(value for value in types if value),
"limit": max(1, min(limit, 500)),
}
if cursor:
params["cursor"] = cursor
value = _data(
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/index",
params=params,
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid Wiki index")
return dict(value)
async def create_wiki_page(self, knowledge_base_id: str, payload: Mapping[str, Any]) -> dict[str, Any]:
value = _data(
await self._request(
"POST",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages",
json=dict(payload),
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid Wiki page")
return dict(value)
async def update_wiki_page(self, knowledge_base_id: str, slug: str, changes: Mapping[str, Any]) -> dict[str, Any]:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
value = _data(
await self._request(
"PUT",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
json=dict(changes),
)
)
return dict(value) if isinstance(value, dict) else {}
async def delete_wiki_page(self, knowledge_base_id: str, slug: str) -> None:
safe_slug = "/".join(quote(part, safe="") for part in slug.split("/"))
await self._request(
"DELETE",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/pages/{safe_slug}",
)
async def search(self, query: str, knowledge_base_ids: Sequence[str]) -> list[dict[str, Any]]:
ids = [str(value) for value in knowledge_base_ids if str(value)]
if not ids:
return []
rows = _as_list(
_data(
await self._request(
"POST",
"/api/v1/knowledge-search",
json={"query": query, "knowledge_base_ids": ids},
)
)
)
normalized: list[dict[str, Any]] = []
for row in rows:
normalized.append(
{
"chunk_id": str(row.get("id") or row.get("chunk_id") or ""),
"content": str(row.get("content") or ""),
"knowledge_id": str(row.get("knowledge_id") or ""),
"knowledge_base_id": str(row.get("knowledge_base_id") or ""),
"title": str(row.get("knowledge_title") or row.get("title") or row.get("knowledge_filename") or ""),
"filename": str(row.get("knowledge_filename") or ""),
"score": float(row.get("score") or 0),
"chunk_index": int(row.get("chunk_index") or 0),
"metadata": row.get("metadata") if isinstance(row.get("metadata"), dict) else {},
}
)
return normalized
async def get_chunk(self, chunk_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/chunks/by-id/{quote(chunk_id, safe='')}"))
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid chunk")
return dict(value)
async def get_chunk_context(self, chunk_id: str, *, expected_knowledge_base_id: str, radius: int = 2) -> list[dict[str, Any]]:
center = await self.get_chunk(chunk_id)
if str(center.get("knowledge_base_id") or "") != expected_knowledge_base_id:
raise WeKnoraError("Chunk does not belong to the authorized knowledge base", status_code=404)
before: list[dict[str, Any]] = []
cursor = center
for _ in range(max(0, min(radius, 4))):
previous_id = str(cursor.get("pre_chunk_id") or "")
if not previous_id:
break
previous = await self.get_chunk(previous_id)
if str(previous.get("knowledge_base_id") or "") != expected_knowledge_base_id:
break
before.append(previous)
cursor = previous
after: list[dict[str, Any]] = []
cursor = center
for _ in range(max(0, min(radius, 4))):
next_id = str(cursor.get("next_chunk_id") or "")
if not next_id:
break
following = await self.get_chunk(next_id)
if str(following.get("knowledge_base_id") or "") != expected_knowledge_base_id:
break
after.append(following)
cursor = following
return [*reversed(before), center, *after]
@@ -0,0 +1,37 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from deerflow.integrations.weknora.local_index.retrieval import (
WikiRetrievalService,
is_fully_vectorized,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = [
"WikiIndexError",
"WikiRetrievalService",
"WikiVectorSearchService",
"WikiSyncService",
"is_fully_vectorized",
]
def __getattr__(name: str) -> Any:
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
from deerflow.integrations.weknora.local_index import retrieval
return getattr(retrieval, name)
if name in {"WikiIndexError", "WikiVectorSearchService"}:
from deerflow.integrations.weknora.local_index import search
return getattr(search, name)
if name == "WikiSyncService":
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
return WikiSyncService
raise AttributeError(name)
@@ -0,0 +1,116 @@
"""Markdown-aware Wiki section chunking."""
from __future__ import annotations
import hashlib
import re
from dataclasses import dataclass
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
@dataclass(frozen=True)
class WikiSection:
section_index: int
heading: str | None
content: str
embedding_text: str
content_hash: str
def _clean_markdown(value: str) -> str:
text = value.replace("\r\n", "\n")
text = _FRONTMATTER_RE.sub("", text)
text = _SCRIPT_STYLE_RE.sub("", text)
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
return re.sub(r"\n{3,}", "\n\n", text).strip()
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
chunks: list[str] = []
buffer = ""
for paragraph in paragraphs:
if len(buffer) + len(paragraph) + 2 <= max_chars:
buffer = f"{buffer}\n\n{paragraph}".strip()
continue
if buffer:
chunks.append(buffer)
buffer = ""
if len(paragraph) <= max_chars:
buffer = paragraph
continue
step = max(1, max_chars - overlap)
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
if buffer:
chunks.append(buffer)
return chunks
def chunk_wiki_page(
page: dict,
*,
max_chars: int,
overlap: int,
min_chars: int,
title_prefix_enabled: bool,
) -> list[WikiSection]:
content = _clean_markdown(str(page.get("content_md") or ""))
heading_stack: list[str] = []
sections: list[tuple[str | None, str]] = []
current_heading: str | None = None
body: list[str] = []
for line in content.splitlines():
match = _HEADING_RE.match(line)
if not match:
body.append(line)
continue
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
level = len(match.group(1))
heading_stack = heading_stack[: level - 1]
heading_stack.append(match.group(2).strip())
current_heading = " / ".join(heading_stack)
body = []
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
if content and not sections:
sections.append((None, content))
summary = _clean_markdown(str(page.get("summary") or ""))
if summary:
sections.insert(0, ("页面摘要", summary))
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
if aliases:
prefix_lines.append(f"别名:{'、'.join(aliases)}")
if page.get("page_type"):
prefix_lines.append(f"类型:{page['page_type']}")
seen: set[str] = set()
result: list[WikiSection] = []
for heading, section_text in sections:
for chunk in _windows(section_text, max_chars, overlap):
normalized = re.sub(r"\s+", " ", chunk).strip()
if len(normalized) < min_chars and len(sections) > 1:
continue
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
if digest in seen:
continue
seen.add(digest)
context = list(prefix_lines) if title_prefix_enabled else []
if heading:
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
# A short article with a short summary must not disappear merely because
# both sections fall below min_chars. Keep one combined, useful vector.
if not result and (content or summary):
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
normalized = re.sub(r"\s+", " ", fallback).strip()
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
prefix = prefix_lines if title_prefix_enabled else []
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
return result
@@ -0,0 +1,480 @@
"""Strict OpenAI-compatible embedding client for the Wiki index."""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from time import monotonic
import httpx
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
logger = logging.getLogger(__name__)
def _upstream_error_detail(response: httpx.Response) -> str:
"""Return a bounded upstream reason without hiding the useful response body."""
try:
payload = response.json()
if isinstance(payload, dict):
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
else:
detail = payload
value = str(detail)
except (ValueError, TypeError):
value = response.text
return " ".join(value.split())[:2000] or "no response body"
class WikiEmbeddingError(RuntimeError):
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
super().__init__(message)
self.category = category
self.retryable = retryable
class StrictWikiEmbeddingClient:
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
self.config = config
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_process: subprocess.Popen[str] | None = None
self._local_lock = asyncio.Lock()
self._local_state = "idle"
self._local_error: str | None = None
self._local_started_at: str | None = None
self._local_ready_at: str | None = None
self._local_warmup_task: asyncio.Task[None] | None = None
@property
def dimensions(self) -> int | None:
return self._resolved_dimensions
@property
def fingerprint(self) -> str:
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
await self.reset_local_encoder()
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
"""Stop only the embedded worker and optionally switch its model path.
This is intentionally independent from the Gateway lifecycle: an
administrator can replace the offline model without restarting or
stopping the rest of DeerFlow.
"""
warmup = self._local_warmup_task
if warmup is not None and not warmup.done():
warmup.cancel()
await asyncio.gather(warmup, return_exceptions=True)
self._local_warmup_task = None
process = self._local_process
self._local_process = None
if process is not None:
await asyncio.to_thread(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
async with self._local_lock:
self._local_state = "starting"
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
try:
values = await asyncio.to_thread(self._embed_local, texts)
except Exception as exc:
self._local_state = "failed"
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
raise
self._local_state = "ready"
self._local_error = None
self._local_ready_at = datetime.now(UTC).isoformat()
return values
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
return []
vectors: list[list[float]] = []
batch_size = max(1, self.config.batch_size)
for start in range(0, len(texts), batch_size):
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
if len(vectors) != len(texts):
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
model_path = self._validated_local_model_path()
if self.config.local_process_isolation:
return self._embed_local_process(texts, model_path)
return self._embed_local_in_process(texts, model_path)
def _validated_local_model_path(self) -> Path:
model_path = self._local_model_path()
if model_path is None:
raise WikiEmbeddingError(
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
category="configuration",
retryable=False,
)
if not model_path.is_dir() or not (
(model_path / "onnx" / "model.onnx").is_file()
or (model_path / "model.onnx").is_file()
):
raise WikiEmbeddingError(
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
category="configuration",
retryable=False,
)
return model_path
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {
"model_name": name,
"threads": self.config.local_threads,
"cache_dir": str(cache_dir),
"providers": ["CPUExecutionProvider"],
"specific_model_path": str(model_path),
"local_files_only": True,
}
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
if len(values) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
process = self._local_process
if process is None or process.poll() is not None:
process = self._start_local_process(model_path)
self._local_process = process
if process.stdin is None or process.stdout is None:
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
try:
process.stdin.write(
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
)
process.stdin.flush()
response = self._read_local_process_message(process, request_id=request_id)
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
self._stop_local_process(process)
self._local_process = None
raise WikiEmbeddingError(
f"本地编码器子进程异常退出:{str(exc)[:500]}",
category="local_process",
retryable=True,
) from exc
if response.get("status") != "ok":
error = str(response.get("error") or "本地编码失败")[:1000]
if process.poll() is not None:
self._local_process = None
raise WikiEmbeddingError(error, category="local_process", retryable=False)
raw_vectors = response.get("vectors")
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
values: list[list[float]] = []
for raw in raw_vectors:
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
return values
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
cache_dir = (
Path(self.config.local_cache_dir).expanduser()
if self.config.local_cache_dir
else Path.home() / ".cache" / "deerflow" / "embeddings"
)
command = [
sys.executable,
"-m",
"deerflow.integrations.weknora.local_index.embedding_worker",
"--model-name",
name,
"--model-path",
str(model_path),
"--cache-dir",
str(cache_dir),
"--threads",
str(self.config.local_threads),
"--batch-size",
str(min(self.config.batch_size, 4)),
]
environment = os.environ.copy()
environment.update(
{
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"HF_DATASETS_OFFLINE": "1",
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
)
creation_flags = 0
if os.name == "nt":
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
)
try:
process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
encoding="utf-8",
bufsize=1,
env=environment,
creationflags=creation_flags,
)
response = self._read_local_process_message(process, ready=True)
except (OSError, ValueError, json.JSONDecodeError) as exc:
if "process" in locals():
self._stop_local_process(process)
raise WikiEmbeddingError(
f"无法启动本地编码器子进程:{str(exc)[:500]}",
category="local_process",
retryable=False,
) from exc
if response.get("status") != "ready":
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
self._stop_local_process(process)
raise WikiEmbeddingError(error, category="local_process", retryable=False)
return process
@staticmethod
def _read_local_process_message(
process: subprocess.Popen[str],
*,
request_id: str | None = None,
ready: bool = False,
) -> dict[str, object]:
if process.stdout is None:
raise ValueError("本地编码器没有输出通道")
for _ in range(20):
line = process.stdout.readline()
if not line:
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
payload = json.loads(line)
if not isinstance(payload, dict):
continue
if ready and payload.get("status") in {"ready", "error"}:
return payload
if request_id is not None and payload.get("id") == request_id:
return payload
raise ValueError("本地编码器返回了无法识别的响应")
@staticmethod
def _stop_local_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
try:
if process.stdin is not None:
process.stdin.write('{"command":"shutdown"}\n')
process.stdin.flush()
process.wait(timeout=2)
except Exception:
process.terminate()
try:
process.wait(timeout=2)
except Exception:
process.kill()
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
if self.config.api_key:
headers["Authorization"] = f"Bearer {self.config.api_key}"
# ``dimensions`` is a local invariant, not a portable request option.
# OpenAI-compatible BGE endpoints commonly reject that extra field
# with HTTP 400 even though they always return a fixed-size vector.
# Keep validating the response against the configured value below.
payload: dict = {"model": self.config.model, "input": texts}
last_error: WikiEmbeddingError | None = None
for attempt in range(self.config.max_retries + 1):
started = monotonic()
try:
response = await (await self._get_client()).post(url, headers=headers, json=payload)
if response.status_code >= 400:
detail = _upstream_error_detail(response)
category = "authentication" if response.status_code in {401, 403} else "upstream"
retryable = response.status_code == 429 or response.status_code >= 500
raise WikiEmbeddingError(
f"Embedding service returned HTTP {response.status_code}: {detail}",
category=category,
retryable=retryable,
)
data = response.json()
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
if len(items) != len(texts):
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
result: list[list[float]] = []
for item in items:
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(vector.size)
result.append(vector.astype(float).tolist())
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
return result
except WikiEmbeddingError as exc:
last_error = exc
except httpx.TimeoutException:
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
except (httpx.HTTPError, ValueError, TypeError) as exc:
reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
last_error = WikiEmbeddingError(
f"Embedding request failed ({type(exc).__name__}): {reason}",
category="request",
retryable=isinstance(exc, httpx.HTTPError),
)
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
logger.error(
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
self.config.model,
len(texts),
last_error.category,
last_error,
)
raise last_error
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
raise last_error or WikiEmbeddingError("Embedding request failed")
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
def local_encoder_status(self) -> dict[str, object]:
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
process_ready = self._local_process is not None and self._local_process.poll() is None
process_failed = self._local_process is not None and not process_ready
state = (
"starting"
if running
else "ready"
if (self._local_model is not None or process_ready)
else "failed"
if process_failed
else self._local_state
)
last_error = self._local_error
if process_failed and not last_error:
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
model_path = self._local_model_path()
model_files_ready = bool(
model_path
and model_path.is_dir()
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
)
return {
"provider": self.config.provider,
"model": self.config.model,
"dimensions": self.dimensions,
"state": state,
"running": running,
"last_error": last_error,
"started_at": self._local_started_at,
"ready_at": self._local_ready_at,
"offline_only": self.config.provider == "local",
"model_path_configured": model_path is not None,
"model_path_exists": bool(model_path and model_path.is_dir()),
"model_files_ready": model_files_ready,
"process_isolation": self.config.local_process_isolation,
"model_path": str(model_path) if model_path is not None else "",
}
def _local_model_path(self) -> Path | None:
raw = self.config.local_model_path.strip() or os.environ.get(
"DEERFLOW_BGE_M3_MODEL_PATH", ""
).strip()
if not raw:
return None
path = Path(raw).expanduser()
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
def start_local_encoder(self) -> dict[str, object]:
"""Warm the embedded CPU encoder without coupling it to app startup."""
if self.config.provider != "local":
raise ValueError("当前 embedding provider 不是 local")
if not self.config.enabled:
raise ValueError("本地编码器未启用")
process_ready = self._local_process is not None and self._local_process.poll() is None
if self._local_model is not None or process_ready:
self._local_state = "ready"
return self.local_encoder_status()
if self._local_warmup_task is None or self._local_warmup_task.done():
self._local_state = "starting"
self._local_error = None
self._local_started_at = datetime.now(UTC).isoformat()
self._local_warmup_task = asyncio.create_task(
self._warm_local_encoder(),
name="llmwiki-local-encoder-warmup",
)
return self.local_encoder_status()
async def _warm_local_encoder(self) -> None:
try:
await self.embed_query("本地编码器启动检查")
except asyncio.CancelledError:
self._local_state = "idle"
raise
except Exception:
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
@@ -0,0 +1,98 @@
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
exist on disk; every offline flag is enforced before FastEmbed is imported.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
def _write(payload: dict[str, object]) -> None:
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def _build_model(args: argparse.Namespace):
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ["HF_DATASETS_OFFLINE"] = "1"
from fastembed import TextEmbedding
if args.model_name == "BAAI/bge-m3" and not any(
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=args.model_name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
return TextEmbedding(
model_name=args.model_name,
threads=args.threads,
cache_dir=args.cache_dir,
providers=["CPUExecutionProvider"],
specific_model_path=args.model_path,
local_files_only=True,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--model-name", required=True)
parser.add_argument("--model-path", required=True)
parser.add_argument("--cache-dir", required=True)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--batch-size", type=int, default=2)
args = parser.parse_args()
model_path = Path(args.model_path)
if not model_path.is_dir():
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
return 2
try:
model = _build_model(args)
except Exception as exc: # noqa: BLE001
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
return 2
_write({"status": "ready"})
for line in sys.stdin:
request: dict[str, object] = {}
try:
request = json.loads(line)
if request.get("command") == "shutdown":
return 0
request_id = str(request.get("id") or "")
texts = request.get("texts")
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
raise ValueError("编码请求格式无效")
vectors = [
value.tolist()
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
]
_write({"id": request_id, "status": "ok", "vectors": vectors})
except Exception as exc: # noqa: BLE001
_write(
{
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
"status": "error",
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
}
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,86 @@
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
from __future__ import annotations
import hashlib
import json
import re
from datetime import datetime
from typing import Any
from urllib.parse import unquote
def normalize_slug(value: Any) -> str:
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
slug = re.sub(r"/{2,}", "/", slug)
if not slug:
raise ValueError("Wiki page slug is empty")
return slug
def slug_hash(slug: str) -> str:
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
def _list(value: Any) -> list:
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
if isinstance(value, str) and value.strip():
return [value.strip()]
return []
def _datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value
raw = str(value or "").strip()
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
status = str(remote.get("status") or "published").strip().lower()
if status not in {"draft", "published", "archived"}:
status = "published"
hash_payload = {
"title": title,
"aliases": aliases,
"summary": summary,
"content_md": content,
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
"status": status,
}
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
return {
"knowledge_base_mapping_id": mapping_id,
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
"slug": slug,
"slug_hash": slug_hash(slug),
"title": title,
"summary": summary,
"content_md": content,
"page_type": hash_payload["page_type"],
"status": status,
"aliases": aliases,
"category_path": category_path,
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
"content_hash": content_hash,
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
}
@@ -0,0 +1,248 @@
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
the raw knowledge/chunk search endpoint, so callers always receive processed
Wiki pages even while a whole knowledge base is awaiting vectorization.
"""
from __future__ import annotations
import asyncio
from typing import Any
from urllib.parse import urlencode
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
"""Return whether a completed full scan produced one coherent active index."""
if not state or not fingerprint:
return False
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
return False
if str(state.get("embedding_fingerprint") or "") != fingerprint:
return False
local_count = int(state.get("local_page_count") or 0)
ready_count = int(state.get("ready_page_count") or 0)
failed_count = int(state.get("failed_page_count") or 0)
vector_count = int(state.get("vector_count") or 0)
if failed_count or ready_count != local_count:
return False
return local_count == 0 or vector_count > 0
class WikiRetrievalService:
"""Search complete indexes locally and incomplete indexes through Wiki API."""
def __init__(
self,
store: LlmWikiIndexStore,
*,
vector_search: WikiVectorSearchService | None,
client: WeKnoraClient | None,
) -> None:
self.store = store
self.vector_search = vector_search
self.client = client
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int = 8,
include_drafts: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {
"query": normalized_query,
"mode": "wiki_api",
"results": [],
"knowledge_base_count": 0,
"partial": False,
"warnings": [],
"fallback_knowledge_base_ids": [],
}
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
results: list[dict[str, Any]] = []
warnings: list[str] = []
vector_used = False
fallback_used = False
if vector_mappings and self.vector_search is not None:
try:
payload = await self.vector_search.search(
normalized_query,
vector_mappings,
top_k_pages=max(top_k_pages, len(vector_mappings)),
include_drafts=include_drafts,
)
results.extend(payload.get("results") or [])
warnings.extend(str(item) for item in payload.get("warnings") or [])
vector_used = True
except WikiIndexError as exc:
# A status row can become stale between classification and
# matrix loading. Wiki API remains a safe Wiki-only fallback.
warnings.append(f"local_vector:{exc.code}")
fallback_mappings.extend(vector_mappings)
vector_mappings = []
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
if fallback_mappings:
if self.client is None:
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
else:
batches = await asyncio.gather(
*(
self._search_remote_mapping(
normalized_query,
mapping,
limit=max(top_k_pages, 1),
include_drafts=include_drafts,
)
for mapping in fallback_mappings
),
return_exceptions=True,
)
for mapping, batch in zip(fallback_mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
else:
results.extend(batch)
fallback_used = True
if warnings and not results and not vector_used and not fallback_used:
raise WikiIndexError(
"WIKI_API_UNAVAILABLE",
"Wiki page retrieval is unavailable",
status_code=502 if self.client is not None else 503,
)
# De-duplicate by the public Wiki citation identity. Vector scores and
# Wiki-API rank scores are both descending relevance hints.
unique: dict[tuple[str, str], dict[str, Any]] = {}
for item in results:
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
current = unique.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
unique[key] = item
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
return {
"query": normalized_query,
"mode": mode,
"results": ranked[:top_k_pages],
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint,
"partial": bool(warnings),
"warnings": warnings,
"fallback_knowledge_base_ids": fallback_ids,
}
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if self.vector_search is None:
return [], list(mappings)
mapping_ids = [str(mapping["id"]) for mapping in mappings]
states = {
str(state["knowledge_base_mapping_id"]): state
for state in await self.store.list_index_status(mapping_ids)
}
fingerprint = self.vector_search.embedding.fingerprint
vector_mappings = [
mapping
for mapping in mappings
if mapping.get("wiki_index_enabled", True)
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
]
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
async def _search_remote_mapping(
self,
query: str,
mapping: dict[str, Any],
*,
limit: int,
include_drafts: bool,
) -> list[dict[str, Any]]:
if self.client is None:
return []
remote_id = str(mapping["weknora_id"])
listing = await self.client.list_wiki_pages(
remote_id,
page=1,
page_size=min(max(limit * 3, 20), 100),
query=query,
)
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
slugs = list(
dict.fromkeys(
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
for item in summaries
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
)
)[: max(limit * 2, limit)]
details = await asyncio.gather(
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
return_exceptions=True,
)
results: list[dict[str, Any]] = []
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
if isinstance(detail, Exception):
continue
try:
page = normalize_wiki_page(str(mapping["id"]), detail)
except (TypeError, ValueError):
continue
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
continue
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
content = str(page.get("content_md") or "")
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
results.append(
{
"kind": "wiki_page",
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"wiki_slug": slug,
"title": page.get("title") or slug,
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at"),
"score": score,
"content": content,
"snippet": content[:360],
"matched_heading": None,
"matched_sections": [
{
"section_index": 0,
"heading": None,
"content": content,
"score": score,
}
],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "wiki_api",
}
)
if len(results) >= limit:
break
return results
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
@@ -0,0 +1,48 @@
"""App-to-harness runtime injection for local Wiki index services."""
from __future__ import annotations
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
_store: LlmWikiIndexStore | None = None
_search: WikiVectorSearchService | None = None
_sync: WikiSyncService | None = None
_retrieval: WikiRetrievalService | None = None
def register_local_wiki_index_runtime(
*,
store: LlmWikiIndexStore | None,
search: WikiVectorSearchService | None,
sync: WikiSyncService | None,
retrieval: WikiRetrievalService | None = None,
) -> None:
global _store, _search, _sync, _retrieval
_store, _search, _sync, _retrieval = store, search, sync, retrieval
def get_local_wiki_index_store() -> LlmWikiIndexStore:
if _store is None:
raise RuntimeError("Local Wiki index store is not initialized")
return _store
def get_local_wiki_search_service() -> WikiVectorSearchService:
if _search is None:
raise RuntimeError("Local Wiki vector search is not initialized")
return _search
def get_local_wiki_sync_service() -> WikiSyncService:
if _sync is None:
raise RuntimeError("Local Wiki sync service is not initialized")
return _sync
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
if _retrieval is None:
raise RuntimeError("Wiki-only retrieval service is not initialized")
return _retrieval
@@ -0,0 +1,316 @@
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
from __future__ import annotations
import asyncio
import hashlib
import re
from collections import OrderedDict, defaultdict
from time import monotonic
from typing import Any
from urllib.parse import urlencode
import numpy as np
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
class WikiIndexError(RuntimeError):
def __init__(self, code: str, message: str, *, status_code: int) -> None:
super().__init__(message)
self.code = code
self.status_code = status_code
class WikiVectorSearchService:
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
self.config = config
self.embedding = embedding
self.cache = cache
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int | None = None,
include_drafts: bool = False,
external: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
fingerprint, candidates, warnings = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=include_drafts,
external=external,
)
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for candidate in candidates:
page = candidate["page"]
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
if len(grouped[key]) < self.config.max_sections_per_page:
grouped[key].append(candidate)
results: list[dict[str, Any]] = []
for sections in grouped.values():
first = sections[0]
page = first["page"]
mapping = first["mapping"]
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
page_score = first["score"] + (0.05 * second_score if second_score else 0)
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
results.append(
{
"kind": "wiki_page",
"id": page["id"],
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page["id"],
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": float(page_score),
"content": "\n\n".join(str(item["section_content"]) for item in sections),
"snippet": str(sections[0]["section_content"])[:360],
"matched_heading": sections[0].get("heading"),
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
}
)
results.sort(key=lambda item: item["score"], reverse=True)
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
async def search_sections(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_sections: int = 10,
external: bool = False,
) -> dict[str, Any]:
"""Return one result per matching Wiki vector section.
The regular ``search`` method intentionally groups several sections
under one Wiki page for citations. Public consumers of the standalone
vector API need the opposite contract: every matched vector is a
separate ranked item, while the complete page and knowledge-base
metadata remains attached to that item.
"""
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
limit = max(1, min(int(top_k_sections), 100))
if not mappings:
return {
"query": normalized_query,
"mode": "vector",
"results": [],
"knowledge_base_count": 0,
"index_fingerprint": self.embedding.fingerprint[:12],
"partial": False,
"warnings": [],
}
fingerprint, candidates, warnings = await self._collect_candidates(
normalized_query,
mappings,
include_drafts=False,
external=external,
section_limit=limit,
)
results: list[dict[str, Any]] = []
for candidate in candidates[:limit]:
page = candidate["page"]
mapping = candidate["mapping"]
results.append(
{
"kind": "wiki_section",
"id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
"section_id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}",
"section_index": int(candidate["section_index"]),
"heading": candidate.get("heading"),
"content": str(candidate.get("section_content") or ""),
"section_content_hash": candidate.get("content_hash") or "",
"score": float(candidate["score"]),
"knowledge_base_id": mapping["id"],
"weknora_knowledge_base_id": mapping.get("weknora_id") or "",
"knowledge_base_name": mapping.get("name") or "",
"knowledge_base_description": mapping.get("description") or "",
"knowledge_base_type": mapping.get("kb_type") or "document",
"publication_status": mapping.get("publication_status") or "",
"published_at": mapping.get("published_at"),
"wiki_page_id": page["id"],
"weknora_wiki_page_id": page.get("remote_page_id"),
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_content": page.get("content_md") or "",
"page_type": page.get("page_type") or "article",
"page_status": page.get("status") or "published",
"page_version": int(page.get("page_version") or 0),
"aliases": page.get("aliases") or [],
"category_path": page.get("category_path") or [],
"parent_slug": page.get("parent_slug"),
"wiki_path": page.get("wiki_path"),
"source_refs": page.get("source_refs") or [],
"chunk_refs": page.get("chunk_refs") or [],
"page_content_hash": page.get("content_hash") or "",
"remote_created_at": page.get("remote_created_at"),
"remote_updated_at": page.get("remote_updated_at"),
"indexed_at": page.get("indexed_at"),
}
)
return {
"query": normalized_query,
"mode": "vector",
"results": results,
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint[:12],
"partial": bool(warnings),
"warnings": warnings,
}
async def _collect_candidates(
self,
query: str,
mappings: list[dict[str, Any]],
*,
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> tuple[str, list[dict[str, Any]], list[str]]:
try:
query_vector = await self._embed_query(query)
except (WikiEmbeddingError, ValueError) as exc:
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
fingerprint = self.embedding.fingerprint
tasks = [
self._search_mapping(
query,
query_vector,
mapping,
fingerprint,
include_drafts=include_drafts,
external=external,
section_limit=section_limit,
)
for mapping in mappings
]
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
return fingerprint, candidates, warnings
async def _embed_query(self, query: str) -> np.ndarray:
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
key = f"{self.embedding.fingerprint}:{digest}"
cached = self._query_vectors.get(key)
if cached is not None and monotonic() - cached[0] <= 300:
self._query_vectors.move_to_end(key)
return cached[1]
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
key = f"{self.embedding.fingerprint}:{digest}"
self._query_vectors[key] = (monotonic(), vector)
self._query_vectors.move_to_end(key)
while len(self._query_vectors) > 256:
self._query_vectors.popitem(last=False)
return vector
async def _search_mapping(
self,
query: str,
vector: np.ndarray,
mapping: dict[str, Any],
fingerprint: str,
*,
include_drafts: bool,
external: bool,
section_limit: int | None = None,
) -> list[dict[str, Any]]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
except ValueError as exc:
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
if snapshot.matrix.size == 0:
return []
if snapshot.dimensions != vector.size:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
eligible = np.asarray(
[
index
for index, row in enumerate(snapshot.rows)
if row["page"].get("status") != "draft" or (include_drafts and not external)
],
dtype=int,
)
if eligible.size == 0:
return []
indices, scores = await asyncio.to_thread(
self._top_indices,
snapshot,
vector,
eligible,
section_limit,
)
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
candidates = []
for index, score in zip(indices, scores, strict=True):
row = snapshot.rows[int(index)]
page = row["page"]
vector_score = float(score)
if vector_score < self.config.min_similarity:
continue
final_score = vector_score
if self.config.keyword_boost_enabled and terms:
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
keyword_score = sum(term in haystack for term in terms) / len(terms)
weight = self.config.keyword_boost_weight
final_score = (1 - weight) * vector_score + weight * keyword_score
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
return candidates
def _top_indices(
self,
snapshot: WikiVectorSnapshot,
vector: np.ndarray,
eligible: np.ndarray,
section_limit: int | None = None,
) -> tuple[np.ndarray, np.ndarray]:
scores = snapshot.matrix[eligible] @ vector
count = min(section_limit or self.config.top_k_sections, scores.size)
if count <= 0:
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
positions = np.argpartition(scores, -count)[-count:]
positions = positions[np.argsort(scores[positions])[::-1]]
return eligible[positions], scores[positions]
@@ -0,0 +1,188 @@
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
from __future__ import annotations
import asyncio
import logging
import platform
from typing import Any
from uuid import uuid4
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
logger = logging.getLogger(__name__)
class WikiSyncLeaseLost(RuntimeError):
pass
def _safe_error(exc: BaseException) -> str:
return f"{type(exc).__name__}: {str(exc)[:1000]}"
class WikiSyncService:
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
self.config = config
self.store = store
self.embedding = embedding
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
# Auto-dimension configurations need one successful response to
# produce a stable fingerprint before lease/rebuild decisions.
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
fingerprint = self.embedding.fingerprint
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
await self.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
prior_state = "rebuild_required"
rebuilding = prior_state == "rebuild_required"
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
if not acquired:
return {"mapping_id": mapping_id, "status": "already_running"}
lease_lost = asyncio.Event()
async def _lease_heartbeat() -> None:
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
while True:
await asyncio.sleep(interval)
try:
renewed = await self.store.renew_sync_lease(
mapping_id,
owner=self.owner,
lease_seconds=self.config.sync_lease_seconds,
)
except Exception:
renewed = False
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
if not renewed:
lease_lost.set()
return
heartbeat = asyncio.create_task(_lease_heartbeat())
def _ensure_lease() -> None:
if lease_lost.is_set():
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
remote_count = 0
changed = 0
failed = 0
try:
remote_id = str(mapping["weknora_id"])
await self.client.get_knowledge_base(remote_id)
page_number = 1
while True:
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
remote_count += len(pages)
reported_total = int(batch.get("total") or batch.get("total_count") or remote_count)
await self.store.update_sync_progress(
mapping_id,
owner=self.owner,
sync_id=sync_id,
remote_page_count=max(remote_count, reported_total),
)
for summary in pages:
try:
_ensure_lease()
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
remote_page = await self.client.get_wiki_page(remote_id, slug)
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
active_fingerprint = self.embedding.fingerprint
if (
not force
and existing
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
continue
sections = (
[]
if page["status"] == "archived"
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
)
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
fingerprint = self.embedding.fingerprint
vectors = []
for section, vector in zip(sections, embeddings, strict=True):
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
vectors.append(
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
)
_ensure_lease()
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
self.cache.invalidate(mapping_id)
changed += 1
except Exception as exc:
if isinstance(exc, WikiSyncLeaseLost):
raise
failed += 1
try:
fallback = normalize_wiki_page(mapping_id, summary)
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
except Exception as record_exc:
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
_ensure_lease()
total_pages = int(batch.get("total_pages") or 1)
if page_number >= total_pages:
break
page_number += 1
_ensure_lease()
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
if deleted:
self.cache.invalidate(mapping_id)
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=not (rebuilding and failed),
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
# Notify even when the remote library is now empty. The global
# assistant base must then remove stale contributions from this
# source instead of retaining deleted Wiki pages forever.
if not failed and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error="Sync cancelled",
)
raise
except Exception as exc:
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
raise
finally:
heartbeat.cancel()
await asyncio.gather(heartbeat, return_exceptions=True)
@@ -0,0 +1,86 @@
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from time import monotonic
from typing import Any
import numpy as np
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
@dataclass(frozen=True)
class WikiVectorSnapshot:
mapping_id: str
index_revision: int
embedding_fingerprint: str
dimensions: int
matrix: np.ndarray
rows: list[dict[str, Any]]
class WikiVectorCache:
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
self.store = store
self.max_knowledge_bases = max_knowledge_bases
self.revision_check_seconds = revision_check_seconds
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
self._checked_at: dict[str, float] = {}
self._locks: dict[str, asyncio.Lock] = {}
def invalidate(self, mapping_id: str) -> None:
self._snapshots.pop(mapping_id, None)
self._checked_at.pop(mapping_id, None)
def clear(self) -> None:
self._snapshots.clear()
self._checked_at.clear()
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
cached = self._snapshots.get(mapping_id)
now = monotonic()
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
self._snapshots.move_to_end(mapping_id)
return cached
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
async with lock:
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
self._checked_at[mapping_id] = monotonic()
cached = self._snapshots.get(mapping_id)
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
if active_fingerprint != fingerprint:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
self._snapshots.move_to_end(mapping_id)
return cached
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
self._snapshots[mapping_id] = snapshot
self._snapshots.move_to_end(mapping_id)
while len(self._snapshots) > self.max_knowledge_bases:
evicted, _ = self._snapshots.popitem(last=False)
self._checked_at.pop(evicted, None)
return snapshot
@staticmethod
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
rows = list(raw.get("rows") or [])
if not rows:
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
dimensions = int(rows[0]["embedding_dimensions"])
vectors = []
valid_rows = []
for row in rows:
if int(row["embedding_dimensions"]) != dimensions:
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
vectors.append(decode_vector(row["vector_blob"], dimensions))
valid_rows.append(row)
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
matrix.setflags(write=False)
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
@@ -0,0 +1,33 @@
"""Validated little-endian normalized float32 Wiki vector codec."""
from __future__ import annotations
import numpy as np
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
vector = np.asarray(values, dtype=np.float32)
if vector.ndim != 1 or vector.size == 0:
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
if dimensions is not None and vector.size != dimensions:
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
if not np.isfinite(vector).all():
raise ValueError("Embedding vector contains NaN or infinity")
norm = float(np.linalg.norm(vector))
if not np.isfinite(norm) or norm <= 0:
raise ValueError("Embedding vector has zero norm")
return np.asarray(vector / norm, dtype="<f4")
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
vector = normalize_vector(values, dimensions=dimensions)
return vector.tobytes(order="C"), int(vector.size)
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
if len(blob) != dimensions * 4:
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
vector = np.frombuffer(blob, dtype="<f4")
if not np.isfinite(vector).all():
raise ValueError("Stored vector contains NaN or infinity")
return vector.copy()
@@ -0,0 +1,45 @@
"""Runtime composition shared by Gateway routes and the read-only agent tool."""
from __future__ import annotations
import httpx
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.config.llmwiki_config import ResolvedLlmWikiRuntime, resolve_llmwiki_runtime
from deerflow.config.system_settings import load_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.persistence.llmwiki import LlmWikiStore
_default_store: LlmWikiStore | None = None
def register_llmwiki_store(store: LlmWikiStore | None) -> None:
global _default_store
_default_store = store
def get_llmwiki_store() -> LlmWikiStore:
if _default_store is None:
raise RuntimeError("LLMWiki store is not initialized")
return _default_store
def get_resolved_llmwiki_runtime(config: AppConfig | None = None) -> ResolvedLlmWikiRuntime:
current = config or get_app_config()
return resolve_llmwiki_runtime(current.llmwiki, load_system_settings().llmwiki)
def build_weknora_client(
runtime: ResolvedLlmWikiRuntime,
*,
http_client: httpx.AsyncClient | None = None,
) -> WeKnoraClient:
if not runtime.weknora_enabled:
raise RuntimeError("WeKnora mode is not enabled")
return WeKnoraClient(
base_url=runtime.api_base_url,
http_client=http_client,
admin_email=runtime.admin_email,
admin_password=runtime.admin_password,
)