feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
+7
@@ -0,0 +1,7 @@
|
||||
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""Markdown-aware Wiki section chunking."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
|
||||
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
|
||||
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
|
||||
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiSection:
|
||||
section_index: int
|
||||
heading: str | None
|
||||
content: str
|
||||
embedding_text: str
|
||||
content_hash: str
|
||||
|
||||
|
||||
def _clean_markdown(value: str) -> str:
|
||||
text = value.replace("\r\n", "\n")
|
||||
text = _FRONTMATTER_RE.sub("", text)
|
||||
text = _SCRIPT_STYLE_RE.sub("", text)
|
||||
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
|
||||
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
|
||||
chunks: list[str] = []
|
||||
buffer = ""
|
||||
for paragraph in paragraphs:
|
||||
if len(buffer) + len(paragraph) + 2 <= max_chars:
|
||||
buffer = f"{buffer}\n\n{paragraph}".strip()
|
||||
continue
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
buffer = ""
|
||||
if len(paragraph) <= max_chars:
|
||||
buffer = paragraph
|
||||
continue
|
||||
step = max(1, max_chars - overlap)
|
||||
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
return chunks
|
||||
|
||||
|
||||
def chunk_wiki_page(
|
||||
page: dict,
|
||||
*,
|
||||
max_chars: int,
|
||||
overlap: int,
|
||||
min_chars: int,
|
||||
title_prefix_enabled: bool,
|
||||
) -> list[WikiSection]:
|
||||
content = _clean_markdown(str(page.get("content_md") or ""))
|
||||
heading_stack: list[str] = []
|
||||
sections: list[tuple[str | None, str]] = []
|
||||
current_heading: str | None = None
|
||||
body: list[str] = []
|
||||
for line in content.splitlines():
|
||||
match = _HEADING_RE.match(line)
|
||||
if not match:
|
||||
body.append(line)
|
||||
continue
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
level = len(match.group(1))
|
||||
heading_stack = heading_stack[: level - 1]
|
||||
heading_stack.append(match.group(2).strip())
|
||||
current_heading = " / ".join(heading_stack)
|
||||
body = []
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
if content and not sections:
|
||||
sections.append((None, content))
|
||||
summary = _clean_markdown(str(page.get("summary") or ""))
|
||||
if summary:
|
||||
sections.insert(0, ("页面摘要", summary))
|
||||
|
||||
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
|
||||
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
|
||||
if aliases:
|
||||
prefix_lines.append(f"别名:{'、'.join(aliases)}")
|
||||
if page.get("page_type"):
|
||||
prefix_lines.append(f"类型:{page['page_type']}")
|
||||
seen: set[str] = set()
|
||||
result: list[WikiSection] = []
|
||||
for heading, section_text in sections:
|
||||
for chunk in _windows(section_text, max_chars, overlap):
|
||||
normalized = re.sub(r"\s+", " ", chunk).strip()
|
||||
if len(normalized) < min_chars and len(sections) > 1:
|
||||
continue
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
if digest in seen:
|
||||
continue
|
||||
seen.add(digest)
|
||||
context = list(prefix_lines) if title_prefix_enabled else []
|
||||
if heading:
|
||||
context.append(f"章节:{heading}")
|
||||
embedding_text = "\n".join([*context, "", chunk]).strip()
|
||||
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
|
||||
return result
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""Strict OpenAI-compatible embedding client for the Wiki index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from time import monotonic
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WikiEmbeddingError(RuntimeError):
|
||||
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
||||
super().__init__(message)
|
||||
self.category = category
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
class StrictWikiEmbeddingClient:
|
||||
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
|
||||
self.config = config
|
||||
self._client = http_client
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
return self._resolved_dimensions
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._owns_client and self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
|
||||
return self._client
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
return []
|
||||
vectors: list[list[float]] = []
|
||||
batch_size = max(1, self.config.batch_size)
|
||||
for start in range(0, len(texts), batch_size):
|
||||
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
|
||||
if len(vectors) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
|
||||
return vectors
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
payload: dict = {"model": self.config.model, "input": texts}
|
||||
if self.config.dimensions:
|
||||
payload["dimensions"] = self.config.dimensions
|
||||
last_error: WikiEmbeddingError | None = None
|
||||
for attempt in range(self.config.max_retries + 1):
|
||||
started = monotonic()
|
||||
try:
|
||||
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
||||
if response.status_code in {401, 403}:
|
||||
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
|
||||
if response.status_code == 429 or response.status_code >= 500:
|
||||
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
||||
if len(items) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
|
||||
result: list[list[float]] = []
|
||||
for item in items:
|
||||
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(vector.size)
|
||||
result.append(vector.astype(float).tolist())
|
||||
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
|
||||
return result
|
||||
except WikiEmbeddingError as exc:
|
||||
last_error = exc
|
||||
except httpx.TimeoutException:
|
||||
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
||||
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
||||
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
|
||||
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
||||
raise last_error
|
||||
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
||||
raise last_error or WikiEmbeddingError("Embedding request failed")
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
result = await self.embed_texts([text])
|
||||
return result[0]
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
def normalize_slug(value: Any) -> str:
|
||||
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
|
||||
slug = re.sub(r"/{2,}", "/", slug)
|
||||
if not slug:
|
||||
raise ValueError("Wiki page slug is empty")
|
||||
return slug
|
||||
|
||||
|
||||
def slug_hash(slug: str) -> str:
|
||||
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _list(value: Any) -> list:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return [value.strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
|
||||
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
|
||||
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
|
||||
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
|
||||
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
|
||||
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
|
||||
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
|
||||
status = str(remote.get("status") or "published").strip().lower()
|
||||
if status not in {"draft", "published", "archived"}:
|
||||
status = "published"
|
||||
hash_payload = {
|
||||
"title": title,
|
||||
"aliases": aliases,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
|
||||
"status": status,
|
||||
}
|
||||
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
return {
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
|
||||
"slug": slug,
|
||||
"slug_hash": slug_hash(slug),
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": hash_payload["page_type"],
|
||||
"status": status,
|
||||
"aliases": aliases,
|
||||
"category_path": category_path,
|
||||
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
|
||||
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
|
||||
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
|
||||
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
|
||||
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
|
||||
"content_hash": content_hash,
|
||||
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
|
||||
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
|
||||
|
||||
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
|
||||
the raw knowledge/chunk search endpoint, so callers always receive processed
|
||||
Wiki pages even while a whole knowledge base is awaiting vectorization.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
|
||||
"""Return whether a completed full scan produced one coherent active index."""
|
||||
|
||||
if not state or not fingerprint:
|
||||
return False
|
||||
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
|
||||
return False
|
||||
if str(state.get("embedding_fingerprint") or "") != fingerprint:
|
||||
return False
|
||||
local_count = int(state.get("local_page_count") or 0)
|
||||
ready_count = int(state.get("ready_page_count") or 0)
|
||||
failed_count = int(state.get("failed_page_count") or 0)
|
||||
vector_count = int(state.get("vector_count") or 0)
|
||||
if failed_count or ready_count != local_count:
|
||||
return False
|
||||
return local_count == 0 or vector_count > 0
|
||||
|
||||
|
||||
class WikiRetrievalService:
|
||||
"""Search complete indexes locally and incomplete indexes through Wiki API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: LlmWikiIndexStore,
|
||||
*,
|
||||
vector_search: WikiVectorSearchService | None,
|
||||
client: WeKnoraClient | None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.vector_search = vector_search
|
||||
self.client = client
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int = 8,
|
||||
include_drafts: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "wiki_api",
|
||||
"results": [],
|
||||
"knowledge_base_count": 0,
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
"fallback_knowledge_base_ids": [],
|
||||
}
|
||||
|
||||
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
|
||||
results: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
vector_used = False
|
||||
fallback_used = False
|
||||
|
||||
if vector_mappings and self.vector_search is not None:
|
||||
try:
|
||||
payload = await self.vector_search.search(
|
||||
normalized_query,
|
||||
vector_mappings,
|
||||
top_k_pages=max(top_k_pages, len(vector_mappings)),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
results.extend(payload.get("results") or [])
|
||||
warnings.extend(str(item) for item in payload.get("warnings") or [])
|
||||
vector_used = True
|
||||
except WikiIndexError as exc:
|
||||
# A status row can become stale between classification and
|
||||
# matrix loading. Wiki API remains a safe Wiki-only fallback.
|
||||
warnings.append(f"local_vector:{exc.code}")
|
||||
fallback_mappings.extend(vector_mappings)
|
||||
vector_mappings = []
|
||||
|
||||
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
|
||||
if fallback_mappings:
|
||||
if self.client is None:
|
||||
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
|
||||
else:
|
||||
batches = await asyncio.gather(
|
||||
*(
|
||||
self._search_remote_mapping(
|
||||
normalized_query,
|
||||
mapping,
|
||||
limit=max(top_k_pages, 1),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
for mapping in fallback_mappings
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for mapping, batch in zip(fallback_mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
|
||||
else:
|
||||
results.extend(batch)
|
||||
fallback_used = True
|
||||
|
||||
if warnings and not results and not vector_used and not fallback_used:
|
||||
raise WikiIndexError(
|
||||
"WIKI_API_UNAVAILABLE",
|
||||
"Wiki page retrieval is unavailable",
|
||||
status_code=502 if self.client is not None else 503,
|
||||
)
|
||||
|
||||
# De-duplicate by the public Wiki citation identity. Vector scores and
|
||||
# Wiki-API rank scores are both descending relevance hints.
|
||||
unique: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for item in results:
|
||||
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
|
||||
current = unique.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
unique[key] = item
|
||||
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
|
||||
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
|
||||
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": mode,
|
||||
"results": ranked[:top_k_pages],
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint,
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"fallback_knowledge_base_ids": fallback_ids,
|
||||
}
|
||||
|
||||
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if self.vector_search is None:
|
||||
return [], list(mappings)
|
||||
mapping_ids = [str(mapping["id"]) for mapping in mappings]
|
||||
states = {
|
||||
str(state["knowledge_base_mapping_id"]): state
|
||||
for state in await self.store.list_index_status(mapping_ids)
|
||||
}
|
||||
fingerprint = self.vector_search.embedding.fingerprint
|
||||
vector_mappings = [
|
||||
mapping
|
||||
for mapping in mappings
|
||||
if mapping.get("wiki_index_enabled", True)
|
||||
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
|
||||
]
|
||||
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
|
||||
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
|
||||
|
||||
async def _search_remote_mapping(
|
||||
self,
|
||||
query: str,
|
||||
mapping: dict[str, Any],
|
||||
*,
|
||||
limit: int,
|
||||
include_drafts: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self.client is None:
|
||||
return []
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
listing = await self.client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=1,
|
||||
page_size=min(max(limit * 3, 20), 100),
|
||||
query=query,
|
||||
)
|
||||
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
slugs = list(
|
||||
dict.fromkeys(
|
||||
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
for item in summaries
|
||||
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
)
|
||||
)[: max(limit * 2, limit)]
|
||||
details = await asyncio.gather(
|
||||
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
|
||||
if isinstance(detail, Exception):
|
||||
continue
|
||||
try:
|
||||
page = normalize_wiki_page(str(mapping["id"]), detail)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
|
||||
continue
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
|
||||
content = str(page.get("content_md") or "")
|
||||
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"wiki_slug": slug,
|
||||
"title": page.get("title") or slug,
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at"),
|
||||
"score": score,
|
||||
"content": content,
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
"matched_sections": [
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"content": content,
|
||||
"score": score,
|
||||
}
|
||||
],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "wiki_api",
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""App-to-harness runtime injection for local Wiki index services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
_store: LlmWikiIndexStore | None = None
|
||||
_search: WikiVectorSearchService | None = None
|
||||
_sync: WikiSyncService | None = None
|
||||
_retrieval: WikiRetrievalService | None = None
|
||||
|
||||
|
||||
def register_local_wiki_index_runtime(
|
||||
*,
|
||||
store: LlmWikiIndexStore | None,
|
||||
search: WikiVectorSearchService | None,
|
||||
sync: WikiSyncService | None,
|
||||
retrieval: WikiRetrievalService | None = None,
|
||||
) -> None:
|
||||
global _store, _search, _sync, _retrieval
|
||||
_store, _search, _sync, _retrieval = store, search, sync, retrieval
|
||||
|
||||
|
||||
def get_local_wiki_index_store() -> LlmWikiIndexStore:
|
||||
if _store is None:
|
||||
raise RuntimeError("Local Wiki index store is not initialized")
|
||||
return _store
|
||||
|
||||
|
||||
def get_local_wiki_search_service() -> WikiVectorSearchService:
|
||||
if _search is None:
|
||||
raise RuntimeError("Local Wiki vector search is not initialized")
|
||||
return _search
|
||||
|
||||
|
||||
def get_local_wiki_sync_service() -> WikiSyncService:
|
||||
if _sync is None:
|
||||
raise RuntimeError("Local Wiki sync service is not initialized")
|
||||
return _sync
|
||||
|
||||
|
||||
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
|
||||
if _retrieval is None:
|
||||
raise RuntimeError("Wiki-only retrieval service is not initialized")
|
||||
return _retrieval
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
from collections import OrderedDict, defaultdict
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
|
||||
class WikiIndexError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, *, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class WikiVectorSearchService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
|
||||
self.config = config
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int | None = None,
|
||||
include_drafts: bool = False,
|
||||
external: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
|
||||
try:
|
||||
query_vector = await self._embed_query(normalized_query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
|
||||
fingerprint = self.embedding.fingerprint
|
||||
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for candidate in candidates:
|
||||
page = candidate["page"]
|
||||
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
|
||||
if len(grouped[key]) < self.config.max_sections_per_page:
|
||||
grouped[key].append(candidate)
|
||||
results: list[dict[str, Any]] = []
|
||||
for sections in grouped.values():
|
||||
first = sections[0]
|
||||
page = first["page"]
|
||||
mapping = first["mapping"]
|
||||
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
|
||||
page_score = first["score"] + (0.05 * second_score if second_score else 0)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page["id"],
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": float(page_score),
|
||||
"content": "\n\n".join(str(item["section_content"]) for item in sections),
|
||||
"snippet": str(sections[0]["section_content"])[:360],
|
||||
"matched_heading": sections[0].get("heading"),
|
||||
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
|
||||
|
||||
async def _embed_query(self, query: str) -> np.ndarray:
|
||||
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
cached = self._query_vectors.get(key)
|
||||
if cached is not None and monotonic() - cached[0] <= 300:
|
||||
self._query_vectors.move_to_end(key)
|
||||
return cached[1]
|
||||
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
self._query_vectors[key] = (monotonic(), vector)
|
||||
self._query_vectors.move_to_end(key)
|
||||
while len(self._query_vectors) > 256:
|
||||
self._query_vectors.popitem(last=False)
|
||||
return vector
|
||||
|
||||
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
except ValueError as exc:
|
||||
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
|
||||
if snapshot.matrix.size == 0:
|
||||
return []
|
||||
if snapshot.dimensions != vector.size:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
eligible = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, row in enumerate(snapshot.rows)
|
||||
if row["page"].get("status") != "draft" or (include_drafts and not external)
|
||||
],
|
||||
dtype=int,
|
||||
)
|
||||
if eligible.size == 0:
|
||||
return []
|
||||
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
|
||||
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
|
||||
candidates = []
|
||||
for index, score in zip(indices, scores, strict=True):
|
||||
row = snapshot.rows[int(index)]
|
||||
page = row["page"]
|
||||
vector_score = float(score)
|
||||
if vector_score < self.config.min_similarity:
|
||||
continue
|
||||
final_score = vector_score
|
||||
if self.config.keyword_boost_enabled and terms:
|
||||
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
|
||||
keyword_score = sum(term in haystack for term in terms) / len(terms)
|
||||
weight = self.config.keyword_boost_weight
|
||||
final_score = (1 - weight) * vector_score + weight * keyword_score
|
||||
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
|
||||
return candidates
|
||||
|
||||
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
||||
scores = snapshot.matrix[eligible] @ vector
|
||||
count = min(self.config.top_k_sections, scores.size)
|
||||
if count <= 0:
|
||||
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
|
||||
positions = np.argpartition(scores, -count)[-count:]
|
||||
positions = positions[np.argsort(scores[positions])[::-1]]
|
||||
return eligible[positions], scores[positions]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WikiSyncLeaseLost(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _safe_error(exc: BaseException) -> str:
|
||||
return f"{type(exc).__name__}: {str(exc)[:1000]}"
|
||||
|
||||
|
||||
class WikiSyncService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
# Auto-dimension configurations need one successful response to
|
||||
# produce a stable fingerprint before lease/rebuild decisions.
|
||||
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
|
||||
fingerprint = self.embedding.fingerprint
|
||||
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
|
||||
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
|
||||
await self.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
prior_state = "rebuild_required"
|
||||
rebuilding = prior_state == "rebuild_required"
|
||||
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
|
||||
if not acquired:
|
||||
return {"mapping_id": mapping_id, "status": "already_running"}
|
||||
lease_lost = asyncio.Event()
|
||||
|
||||
async def _lease_heartbeat() -> None:
|
||||
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
renewed = await self.store.renew_sync_lease(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.config.sync_lease_seconds,
|
||||
)
|
||||
except Exception:
|
||||
renewed = False
|
||||
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
|
||||
if not renewed:
|
||||
lease_lost.set()
|
||||
return
|
||||
|
||||
heartbeat = asyncio.create_task(_lease_heartbeat())
|
||||
|
||||
def _ensure_lease() -> None:
|
||||
if lease_lost.is_set():
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
|
||||
|
||||
remote_count = 0
|
||||
changed = 0
|
||||
failed = 0
|
||||
try:
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
await self.client.get_knowledge_base(remote_id)
|
||||
page_number = 1
|
||||
while True:
|
||||
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
|
||||
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
|
||||
remote_count += len(pages)
|
||||
for summary in pages:
|
||||
try:
|
||||
_ensure_lease()
|
||||
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
|
||||
remote_page = await self.client.get_wiki_page(remote_id, slug)
|
||||
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
|
||||
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
|
||||
active_fingerprint = self.embedding.fingerprint
|
||||
if (
|
||||
not force
|
||||
and existing
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and not existing.get("is_remote_deleted")
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
continue
|
||||
sections = (
|
||||
[]
|
||||
if page["status"] == "archived"
|
||||
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
|
||||
)
|
||||
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
|
||||
fingerprint = self.embedding.fingerprint
|
||||
vectors = []
|
||||
for section, vector in zip(sections, embeddings, strict=True):
|
||||
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
|
||||
vectors.append(
|
||||
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
|
||||
)
|
||||
_ensure_lease()
|
||||
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
|
||||
self.cache.invalidate(mapping_id)
|
||||
changed += 1
|
||||
except Exception as exc:
|
||||
if isinstance(exc, WikiSyncLeaseLost):
|
||||
raise
|
||||
failed += 1
|
||||
try:
|
||||
fallback = normalize_wiki_page(mapping_id, summary)
|
||||
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
|
||||
except Exception as record_exc:
|
||||
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
|
||||
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
|
||||
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
_ensure_lease()
|
||||
total_pages = int(batch.get("total_pages") or 1)
|
||||
if page_number >= total_pages:
|
||||
break
|
||||
page_number += 1
|
||||
_ensure_lease()
|
||||
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
|
||||
if deleted:
|
||||
self.cache.invalidate(mapping_id)
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=not (rebuilding and failed),
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
except asyncio.CancelledError:
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error="Sync cancelled",
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
|
||||
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
await asyncio.gather(heartbeat, return_exceptions=True)
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiVectorSnapshot:
|
||||
mapping_id: str
|
||||
index_revision: int
|
||||
embedding_fingerprint: str
|
||||
dimensions: int
|
||||
matrix: np.ndarray
|
||||
rows: list[dict[str, Any]]
|
||||
|
||||
|
||||
class WikiVectorCache:
|
||||
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
|
||||
self.store = store
|
||||
self.max_knowledge_bases = max_knowledge_bases
|
||||
self.revision_check_seconds = revision_check_seconds
|
||||
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
|
||||
self._checked_at: dict[str, float] = {}
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
def invalidate(self, mapping_id: str) -> None:
|
||||
self._snapshots.pop(mapping_id, None)
|
||||
self._checked_at.pop(mapping_id, None)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._snapshots.clear()
|
||||
self._checked_at.clear()
|
||||
|
||||
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
now = monotonic()
|
||||
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
|
||||
async with lock:
|
||||
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
|
||||
self._checked_at[mapping_id] = monotonic()
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
|
||||
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
|
||||
if active_fingerprint != fingerprint:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
|
||||
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
|
||||
self._snapshots[mapping_id] = snapshot
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
while len(self._snapshots) > self.max_knowledge_bases:
|
||||
evicted, _ = self._snapshots.popitem(last=False)
|
||||
self._checked_at.pop(evicted, None)
|
||||
return snapshot
|
||||
|
||||
@staticmethod
|
||||
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
|
||||
rows = list(raw.get("rows") or [])
|
||||
if not rows:
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
|
||||
dimensions = int(rows[0]["embedding_dimensions"])
|
||||
vectors = []
|
||||
valid_rows = []
|
||||
for row in rows:
|
||||
if int(row["embedding_dimensions"]) != dimensions:
|
||||
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
|
||||
vectors.append(decode_vector(row["vector_blob"], dimensions))
|
||||
valid_rows.append(row)
|
||||
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
|
||||
matrix.setflags(write=False)
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Validated little-endian normalized float32 Wiki vector codec."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
|
||||
vector = np.asarray(values, dtype=np.float32)
|
||||
if vector.ndim != 1 or vector.size == 0:
|
||||
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
|
||||
if dimensions is not None and vector.size != dimensions:
|
||||
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Embedding vector contains NaN or infinity")
|
||||
norm = float(np.linalg.norm(vector))
|
||||
if not np.isfinite(norm) or norm <= 0:
|
||||
raise ValueError("Embedding vector has zero norm")
|
||||
return np.asarray(vector / norm, dtype="<f4")
|
||||
|
||||
|
||||
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
|
||||
vector = normalize_vector(values, dimensions=dimensions)
|
||||
return vector.tobytes(order="C"), int(vector.size)
|
||||
|
||||
|
||||
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
|
||||
if len(blob) != dimensions * 4:
|
||||
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
|
||||
vector = np.frombuffer(blob, dtype="<f4")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Stored vector contains NaN or infinity")
|
||||
return vector.copy()
|
||||
Reference in new issue
Block a user