feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -0,0 +1,7 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
@@ -0,0 +1,108 @@
"""Markdown-aware Wiki section chunking."""
from __future__ import annotations
import hashlib
import re
from dataclasses import dataclass
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
@dataclass(frozen=True)
class WikiSection:
section_index: int
heading: str | None
content: str
embedding_text: str
content_hash: str
def _clean_markdown(value: str) -> str:
text = value.replace("\r\n", "\n")
text = _FRONTMATTER_RE.sub("", text)
text = _SCRIPT_STYLE_RE.sub("", text)
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
return re.sub(r"\n{3,}", "\n\n", text).strip()
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
chunks: list[str] = []
buffer = ""
for paragraph in paragraphs:
if len(buffer) + len(paragraph) + 2 <= max_chars:
buffer = f"{buffer}\n\n{paragraph}".strip()
continue
if buffer:
chunks.append(buffer)
buffer = ""
if len(paragraph) <= max_chars:
buffer = paragraph
continue
step = max(1, max_chars - overlap)
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
if buffer:
chunks.append(buffer)
return chunks
def chunk_wiki_page(
page: dict,
*,
max_chars: int,
overlap: int,
min_chars: int,
title_prefix_enabled: bool,
) -> list[WikiSection]:
content = _clean_markdown(str(page.get("content_md") or ""))
heading_stack: list[str] = []
sections: list[tuple[str | None, str]] = []
current_heading: str | None = None
body: list[str] = []
for line in content.splitlines():
match = _HEADING_RE.match(line)
if not match:
body.append(line)
continue
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
level = len(match.group(1))
heading_stack = heading_stack[: level - 1]
heading_stack.append(match.group(2).strip())
current_heading = " / ".join(heading_stack)
body = []
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
if content and not sections:
sections.append((None, content))
summary = _clean_markdown(str(page.get("summary") or ""))
if summary:
sections.insert(0, ("页面摘要", summary))
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
if aliases:
prefix_lines.append(f"别名:{'、'.join(aliases)}")
if page.get("page_type"):
prefix_lines.append(f"类型:{page['page_type']}")
seen: set[str] = set()
result: list[WikiSection] = []
for heading, section_text in sections:
for chunk in _windows(section_text, max_chars, overlap):
normalized = re.sub(r"\s+", " ", chunk).strip()
if len(normalized) < min_chars and len(sections) > 1:
continue
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
if digest in seen:
continue
seen.add(digest)
context = list(prefix_lines) if title_prefix_enabled else []
if heading:
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
return result
@@ -0,0 +1,107 @@
"""Strict OpenAI-compatible embedding client for the Wiki index."""
from __future__ import annotations
import asyncio
import hashlib
import logging
from time import monotonic
import httpx
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
logger = logging.getLogger(__name__)
class WikiEmbeddingError(RuntimeError):
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
super().__init__(message)
self.category = category
self.retryable = retryable
class StrictWikiEmbeddingClient:
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
self.config = config
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
@property
def dimensions(self) -> int | None:
return self._resolved_dimensions
@property
def fingerprint(self) -> str:
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
return []
vectors: list[list[float]] = []
batch_size = max(1, self.config.batch_size)
for start in range(0, len(texts), batch_size):
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
if len(vectors) != len(texts):
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
if self.config.api_key:
headers["Authorization"] = f"Bearer {self.config.api_key}"
payload: dict = {"model": self.config.model, "input": texts}
if self.config.dimensions:
payload["dimensions"] = self.config.dimensions
last_error: WikiEmbeddingError | None = None
for attempt in range(self.config.max_retries + 1):
started = monotonic()
try:
response = await (await self._get_client()).post(url, headers=headers, json=payload)
if response.status_code in {401, 403}:
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
if response.status_code == 429 or response.status_code >= 500:
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
response.raise_for_status()
data = response.json()
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
if len(items) != len(texts):
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
result: list[list[float]] = []
for item in items:
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(vector.size)
result.append(vector.astype(float).tolist())
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
return result
except WikiEmbeddingError as exc:
last_error = exc
except httpx.TimeoutException:
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
except (httpx.HTTPError, ValueError, TypeError) as exc:
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
raise last_error
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
raise last_error or WikiEmbeddingError("Embedding request failed")
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
@@ -0,0 +1,86 @@
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
from __future__ import annotations
import hashlib
import json
import re
from datetime import datetime
from typing import Any
from urllib.parse import unquote
def normalize_slug(value: Any) -> str:
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
slug = re.sub(r"/{2,}", "/", slug)
if not slug:
raise ValueError("Wiki page slug is empty")
return slug
def slug_hash(slug: str) -> str:
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
def _list(value: Any) -> list:
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
if isinstance(value, str) and value.strip():
return [value.strip()]
return []
def _datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value
raw = str(value or "").strip()
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
status = str(remote.get("status") or "published").strip().lower()
if status not in {"draft", "published", "archived"}:
status = "published"
hash_payload = {
"title": title,
"aliases": aliases,
"summary": summary,
"content_md": content,
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
"status": status,
}
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
return {
"knowledge_base_mapping_id": mapping_id,
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
"slug": slug,
"slug_hash": slug_hash(slug),
"title": title,
"summary": summary,
"content_md": content,
"page_type": hash_payload["page_type"],
"status": status,
"aliases": aliases,
"category_path": category_path,
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
"content_hash": content_hash,
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
}
@@ -0,0 +1,248 @@
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
the raw knowledge/chunk search endpoint, so callers always receive processed
Wiki pages even while a whole knowledge base is awaiting vectorization.
"""
from __future__ import annotations
import asyncio
from typing import Any
from urllib.parse import urlencode
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
"""Return whether a completed full scan produced one coherent active index."""
if not state or not fingerprint:
return False
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
return False
if str(state.get("embedding_fingerprint") or "") != fingerprint:
return False
local_count = int(state.get("local_page_count") or 0)
ready_count = int(state.get("ready_page_count") or 0)
failed_count = int(state.get("failed_page_count") or 0)
vector_count = int(state.get("vector_count") or 0)
if failed_count or ready_count != local_count:
return False
return local_count == 0 or vector_count > 0
class WikiRetrievalService:
"""Search complete indexes locally and incomplete indexes through Wiki API."""
def __init__(
self,
store: LlmWikiIndexStore,
*,
vector_search: WikiVectorSearchService | None,
client: WeKnoraClient | None,
) -> None:
self.store = store
self.vector_search = vector_search
self.client = client
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int = 8,
include_drafts: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {
"query": normalized_query,
"mode": "wiki_api",
"results": [],
"knowledge_base_count": 0,
"partial": False,
"warnings": [],
"fallback_knowledge_base_ids": [],
}
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
results: list[dict[str, Any]] = []
warnings: list[str] = []
vector_used = False
fallback_used = False
if vector_mappings and self.vector_search is not None:
try:
payload = await self.vector_search.search(
normalized_query,
vector_mappings,
top_k_pages=max(top_k_pages, len(vector_mappings)),
include_drafts=include_drafts,
)
results.extend(payload.get("results") or [])
warnings.extend(str(item) for item in payload.get("warnings") or [])
vector_used = True
except WikiIndexError as exc:
# A status row can become stale between classification and
# matrix loading. Wiki API remains a safe Wiki-only fallback.
warnings.append(f"local_vector:{exc.code}")
fallback_mappings.extend(vector_mappings)
vector_mappings = []
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
if fallback_mappings:
if self.client is None:
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
else:
batches = await asyncio.gather(
*(
self._search_remote_mapping(
normalized_query,
mapping,
limit=max(top_k_pages, 1),
include_drafts=include_drafts,
)
for mapping in fallback_mappings
),
return_exceptions=True,
)
for mapping, batch in zip(fallback_mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
else:
results.extend(batch)
fallback_used = True
if warnings and not results and not vector_used and not fallback_used:
raise WikiIndexError(
"WIKI_API_UNAVAILABLE",
"Wiki page retrieval is unavailable",
status_code=502 if self.client is not None else 503,
)
# De-duplicate by the public Wiki citation identity. Vector scores and
# Wiki-API rank scores are both descending relevance hints.
unique: dict[tuple[str, str], dict[str, Any]] = {}
for item in results:
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
current = unique.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
unique[key] = item
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
return {
"query": normalized_query,
"mode": mode,
"results": ranked[:top_k_pages],
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint,
"partial": bool(warnings),
"warnings": warnings,
"fallback_knowledge_base_ids": fallback_ids,
}
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if self.vector_search is None:
return [], list(mappings)
mapping_ids = [str(mapping["id"]) for mapping in mappings]
states = {
str(state["knowledge_base_mapping_id"]): state
for state in await self.store.list_index_status(mapping_ids)
}
fingerprint = self.vector_search.embedding.fingerprint
vector_mappings = [
mapping
for mapping in mappings
if mapping.get("wiki_index_enabled", True)
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
]
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
async def _search_remote_mapping(
self,
query: str,
mapping: dict[str, Any],
*,
limit: int,
include_drafts: bool,
) -> list[dict[str, Any]]:
if self.client is None:
return []
remote_id = str(mapping["weknora_id"])
listing = await self.client.list_wiki_pages(
remote_id,
page=1,
page_size=min(max(limit * 3, 20), 100),
query=query,
)
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
slugs = list(
dict.fromkeys(
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
for item in summaries
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
)
)[: max(limit * 2, limit)]
details = await asyncio.gather(
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
return_exceptions=True,
)
results: list[dict[str, Any]] = []
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
if isinstance(detail, Exception):
continue
try:
page = normalize_wiki_page(str(mapping["id"]), detail)
except (TypeError, ValueError):
continue
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
continue
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
content = str(page.get("content_md") or "")
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
results.append(
{
"kind": "wiki_page",
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"wiki_slug": slug,
"title": page.get("title") or slug,
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at"),
"score": score,
"content": content,
"snippet": content[:360],
"matched_heading": None,
"matched_sections": [
{
"section_index": 0,
"heading": None,
"content": content,
"score": score,
}
],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "wiki_api",
}
)
if len(results) >= limit:
break
return results
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
@@ -0,0 +1,48 @@
"""App-to-harness runtime injection for local Wiki index services."""
from __future__ import annotations
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
_store: LlmWikiIndexStore | None = None
_search: WikiVectorSearchService | None = None
_sync: WikiSyncService | None = None
_retrieval: WikiRetrievalService | None = None
def register_local_wiki_index_runtime(
*,
store: LlmWikiIndexStore | None,
search: WikiVectorSearchService | None,
sync: WikiSyncService | None,
retrieval: WikiRetrievalService | None = None,
) -> None:
global _store, _search, _sync, _retrieval
_store, _search, _sync, _retrieval = store, search, sync, retrieval
def get_local_wiki_index_store() -> LlmWikiIndexStore:
if _store is None:
raise RuntimeError("Local Wiki index store is not initialized")
return _store
def get_local_wiki_search_service() -> WikiVectorSearchService:
if _search is None:
raise RuntimeError("Local Wiki vector search is not initialized")
return _search
def get_local_wiki_sync_service() -> WikiSyncService:
if _sync is None:
raise RuntimeError("Local Wiki sync service is not initialized")
return _sync
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
if _retrieval is None:
raise RuntimeError("Wiki-only retrieval service is not initialized")
return _retrieval
@@ -0,0 +1,176 @@
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
from __future__ import annotations
import asyncio
import hashlib
import re
from collections import OrderedDict, defaultdict
from time import monotonic
from typing import Any
from urllib.parse import urlencode
import numpy as np
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
class WikiIndexError(RuntimeError):
def __init__(self, code: str, message: str, *, status_code: int) -> None:
super().__init__(message)
self.code = code
self.status_code = status_code
class WikiVectorSearchService:
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
self.config = config
self.embedding = embedding
self.cache = cache
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int | None = None,
include_drafts: bool = False,
external: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
try:
query_vector = await self._embed_query(normalized_query)
except (WikiEmbeddingError, ValueError) as exc:
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
fingerprint = self.embedding.fingerprint
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for candidate in candidates:
page = candidate["page"]
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
if len(grouped[key]) < self.config.max_sections_per_page:
grouped[key].append(candidate)
results: list[dict[str, Any]] = []
for sections in grouped.values():
first = sections[0]
page = first["page"]
mapping = first["mapping"]
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
page_score = first["score"] + (0.05 * second_score if second_score else 0)
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
results.append(
{
"kind": "wiki_page",
"id": page["id"],
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page["id"],
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": float(page_score),
"content": "\n\n".join(str(item["section_content"]) for item in sections),
"snippet": str(sections[0]["section_content"])[:360],
"matched_heading": sections[0].get("heading"),
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
}
)
results.sort(key=lambda item: item["score"], reverse=True)
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
async def _embed_query(self, query: str) -> np.ndarray:
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
key = f"{self.embedding.fingerprint}:{digest}"
cached = self._query_vectors.get(key)
if cached is not None and monotonic() - cached[0] <= 300:
self._query_vectors.move_to_end(key)
return cached[1]
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
key = f"{self.embedding.fingerprint}:{digest}"
self._query_vectors[key] = (monotonic(), vector)
self._query_vectors.move_to_end(key)
while len(self._query_vectors) > 256:
self._query_vectors.popitem(last=False)
return vector
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
except ValueError as exc:
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
if snapshot.matrix.size == 0:
return []
if snapshot.dimensions != vector.size:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
eligible = np.asarray(
[
index
for index, row in enumerate(snapshot.rows)
if row["page"].get("status") != "draft" or (include_drafts and not external)
],
dtype=int,
)
if eligible.size == 0:
return []
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
candidates = []
for index, score in zip(indices, scores, strict=True):
row = snapshot.rows[int(index)]
page = row["page"]
vector_score = float(score)
if vector_score < self.config.min_similarity:
continue
final_score = vector_score
if self.config.keyword_boost_enabled and terms:
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
keyword_score = sum(term in haystack for term in terms) / len(terms)
weight = self.config.keyword_boost_weight
final_score = (1 - weight) * vector_score + weight * keyword_score
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
return candidates
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
scores = snapshot.matrix[eligible] @ vector
count = min(self.config.top_k_sections, scores.size)
if count <= 0:
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
positions = np.argpartition(scores, -count)[-count:]
positions = positions[np.argsort(scores[positions])[::-1]]
return eligible[positions], scores[positions]
@@ -0,0 +1,173 @@
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
from __future__ import annotations
import asyncio
import logging
import platform
from typing import Any
from uuid import uuid4
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
logger = logging.getLogger(__name__)
class WikiSyncLeaseLost(RuntimeError):
pass
def _safe_error(exc: BaseException) -> str:
return f"{type(exc).__name__}: {str(exc)[:1000]}"
class WikiSyncService:
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
self.config = config
self.store = store
self.embedding = embedding
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
# Auto-dimension configurations need one successful response to
# produce a stable fingerprint before lease/rebuild decisions.
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
fingerprint = self.embedding.fingerprint
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
await self.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
prior_state = "rebuild_required"
rebuilding = prior_state == "rebuild_required"
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
if not acquired:
return {"mapping_id": mapping_id, "status": "already_running"}
lease_lost = asyncio.Event()
async def _lease_heartbeat() -> None:
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
while True:
await asyncio.sleep(interval)
try:
renewed = await self.store.renew_sync_lease(
mapping_id,
owner=self.owner,
lease_seconds=self.config.sync_lease_seconds,
)
except Exception:
renewed = False
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
if not renewed:
lease_lost.set()
return
heartbeat = asyncio.create_task(_lease_heartbeat())
def _ensure_lease() -> None:
if lease_lost.is_set():
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
remote_count = 0
changed = 0
failed = 0
try:
remote_id = str(mapping["weknora_id"])
await self.client.get_knowledge_base(remote_id)
page_number = 1
while True:
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
remote_count += len(pages)
for summary in pages:
try:
_ensure_lease()
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
remote_page = await self.client.get_wiki_page(remote_id, slug)
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
active_fingerprint = self.embedding.fingerprint
if (
not force
and existing
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
continue
sections = (
[]
if page["status"] == "archived"
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
)
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
fingerprint = self.embedding.fingerprint
vectors = []
for section, vector in zip(sections, embeddings, strict=True):
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
vectors.append(
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
)
_ensure_lease()
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
self.cache.invalidate(mapping_id)
changed += 1
except Exception as exc:
if isinstance(exc, WikiSyncLeaseLost):
raise
failed += 1
try:
fallback = normalize_wiki_page(mapping_id, summary)
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
except Exception as record_exc:
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
_ensure_lease()
total_pages = int(batch.get("total_pages") or 1)
if page_number >= total_pages:
break
page_number += 1
_ensure_lease()
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
if deleted:
self.cache.invalidate(mapping_id)
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=not (rebuilding and failed),
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error="Sync cancelled",
)
raise
except Exception as exc:
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
raise
finally:
heartbeat.cancel()
await asyncio.gather(heartbeat, return_exceptions=True)
@@ -0,0 +1,86 @@
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from time import monotonic
from typing import Any
import numpy as np
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
@dataclass(frozen=True)
class WikiVectorSnapshot:
mapping_id: str
index_revision: int
embedding_fingerprint: str
dimensions: int
matrix: np.ndarray
rows: list[dict[str, Any]]
class WikiVectorCache:
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
self.store = store
self.max_knowledge_bases = max_knowledge_bases
self.revision_check_seconds = revision_check_seconds
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
self._checked_at: dict[str, float] = {}
self._locks: dict[str, asyncio.Lock] = {}
def invalidate(self, mapping_id: str) -> None:
self._snapshots.pop(mapping_id, None)
self._checked_at.pop(mapping_id, None)
def clear(self) -> None:
self._snapshots.clear()
self._checked_at.clear()
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
cached = self._snapshots.get(mapping_id)
now = monotonic()
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
self._snapshots.move_to_end(mapping_id)
return cached
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
async with lock:
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
self._checked_at[mapping_id] = monotonic()
cached = self._snapshots.get(mapping_id)
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
if active_fingerprint != fingerprint:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
self._snapshots.move_to_end(mapping_id)
return cached
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
self._snapshots[mapping_id] = snapshot
self._snapshots.move_to_end(mapping_id)
while len(self._snapshots) > self.max_knowledge_bases:
evicted, _ = self._snapshots.popitem(last=False)
self._checked_at.pop(evicted, None)
return snapshot
@staticmethod
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
rows = list(raw.get("rows") or [])
if not rows:
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
dimensions = int(rows[0]["embedding_dimensions"])
vectors = []
valid_rows = []
for row in rows:
if int(row["embedding_dimensions"]) != dimensions:
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
vectors.append(decode_vector(row["vector_blob"], dimensions))
valid_rows.append(row)
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
matrix.setflags(write=False)
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
@@ -0,0 +1,33 @@
"""Validated little-endian normalized float32 Wiki vector codec."""
from __future__ import annotations
import numpy as np
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
vector = np.asarray(values, dtype=np.float32)
if vector.ndim != 1 or vector.size == 0:
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
if dimensions is not None and vector.size != dimensions:
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
if not np.isfinite(vector).all():
raise ValueError("Embedding vector contains NaN or infinity")
norm = float(np.linalg.norm(vector))
if not np.isfinite(norm) or norm <= 0:
raise ValueError("Embedding vector has zero norm")
return np.asarray(vector / norm, dtype="<f4")
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
vector = normalize_vector(values, dimensions=dimensions)
return vector.tobytes(order="C"), int(vector.size)
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
if len(blob) != dimensions * 4:
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
vector = np.frombuffer(blob, dtype="<f4")
if not np.isfinite(vector).all():
raise ValueError("Stored vector contains NaN or infinity")
return vector.copy()