feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -10,7 +10,7 @@ from __future__ import annotations
from dataclasses import dataclass
from urllib.parse import urlsplit, urlunsplit
from pydantic import BaseModel, ConfigDict, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
def normalize_service_url(value: str | None) -> str:
@@ -37,18 +37,12 @@ class WeKnoraConnectionConfig(BaseModel):
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
admin_email: str = Field(
default="",
description=(
"Optional WeKnora administrator email used by DeerFlow's server-side "
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
),
description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
admin_password: str = Field(
default="",
repr=False,
description=(
"Optional WeKnora administrator password used by DeerFlow's server-side "
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
),
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
model_config = ConfigDict(extra="ignore")
@@ -58,10 +52,107 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
dimensions: int | None = Field(default=1024, ge=1, le=65536)
batch_size: int = Field(default=32, ge=1, le=512)
timeout_seconds: float = Field(default=30, gt=0, le=300)
max_retries: int = Field(default=3, ge=0, le=10)
model_config = ConfigDict(extra="ignore")
@field_validator("provider")
@classmethod
def _validate_provider(cls, value: str) -> str:
normalized = value.strip().lower()
if normalized not in {"openai", "openai_compatible"}:
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
return normalized
@field_validator("base_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiExternalApiConfig(BaseModel):
enabled: bool = False
api_key: str = Field(default="", repr=False)
previous_api_key: str = Field(default="", repr=False)
max_query_chars: int = Field(default=1000, ge=1, le=10000)
max_knowledge_bases: int = Field(default=20, ge=1, le=100)
max_results: int = Field(default=50, ge=1, le=100)
default_results: int = Field(default=10, ge=1, le=100)
include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
requests_per_minute: int = Field(default=60, ge=1, le=100000)
model_config = ConfigDict(extra="ignore")
@model_validator(mode="after")
def _validate_enabled_key(self):
if self.enabled and not self.api_key.strip():
raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
if self.default_results > self.max_results:
raise ValueError("external_api.default_results must not exceed max_results")
return self
class LocalWikiIndexConfig(BaseModel):
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
enabled: bool = False
sync_enabled: bool = False
internal_search_enabled: bool = False
reference_ui_enabled: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
sync_max_retries: int = Field(default=3, ge=0, le=10)
sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
chunk_min_chars: int = Field(default=80, ge=1, le=10000)
title_prefix_enabled: bool = True
embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
engine: str = "numpy_exact"
strict_vector: bool = True
top_k_sections: int = Field(default=30, ge=1, le=1000)
top_k_pages: int = Field(default=8, ge=1, le=100)
max_sections_per_page: int = Field(default=2, ge=1, le=20)
min_similarity: float = Field(default=0.35, ge=-1, le=1)
keyword_boost_enabled: bool = False
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
model_config = ConfigDict(extra="ignore")
@model_validator(mode="after")
def _validate_index(self):
if self.chunk_overlap_chars >= self.chunk_max_chars:
raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
if self.chunk_min_chars > self.chunk_max_chars:
raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
if self.top_k_pages > self.top_k_sections:
raise ValueError("top_k_pages must not exceed top_k_sections")
if self.engine != "numpy_exact":
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
if self.enabled and not self.embedding.enabled:
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
return self
class LlmWikiConfig(BaseModel):
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
model_config = ConfigDict(extra="ignore")
@@ -0,0 +1,7 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
@@ -0,0 +1,108 @@
"""Markdown-aware Wiki section chunking."""
from __future__ import annotations
import hashlib
import re
from dataclasses import dataclass
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
@dataclass(frozen=True)
class WikiSection:
section_index: int
heading: str | None
content: str
embedding_text: str
content_hash: str
def _clean_markdown(value: str) -> str:
text = value.replace("\r\n", "\n")
text = _FRONTMATTER_RE.sub("", text)
text = _SCRIPT_STYLE_RE.sub("", text)
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
return re.sub(r"\n{3,}", "\n\n", text).strip()
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
chunks: list[str] = []
buffer = ""
for paragraph in paragraphs:
if len(buffer) + len(paragraph) + 2 <= max_chars:
buffer = f"{buffer}\n\n{paragraph}".strip()
continue
if buffer:
chunks.append(buffer)
buffer = ""
if len(paragraph) <= max_chars:
buffer = paragraph
continue
step = max(1, max_chars - overlap)
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
if buffer:
chunks.append(buffer)
return chunks
def chunk_wiki_page(
page: dict,
*,
max_chars: int,
overlap: int,
min_chars: int,
title_prefix_enabled: bool,
) -> list[WikiSection]:
content = _clean_markdown(str(page.get("content_md") or ""))
heading_stack: list[str] = []
sections: list[tuple[str | None, str]] = []
current_heading: str | None = None
body: list[str] = []
for line in content.splitlines():
match = _HEADING_RE.match(line)
if not match:
body.append(line)
continue
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
level = len(match.group(1))
heading_stack = heading_stack[: level - 1]
heading_stack.append(match.group(2).strip())
current_heading = " / ".join(heading_stack)
body = []
if any(item.strip() for item in body):
sections.append((current_heading, "\n".join(body).strip()))
if content and not sections:
sections.append((None, content))
summary = _clean_markdown(str(page.get("summary") or ""))
if summary:
sections.insert(0, ("页面摘要", summary))
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
if aliases:
prefix_lines.append(f"别名:{'、'.join(aliases)}")
if page.get("page_type"):
prefix_lines.append(f"类型:{page['page_type']}")
seen: set[str] = set()
result: list[WikiSection] = []
for heading, section_text in sections:
for chunk in _windows(section_text, max_chars, overlap):
normalized = re.sub(r"\s+", " ", chunk).strip()
if len(normalized) < min_chars and len(sections) > 1:
continue
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
if digest in seen:
continue
seen.add(digest)
context = list(prefix_lines) if title_prefix_enabled else []
if heading:
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
return result
@@ -0,0 +1,107 @@
"""Strict OpenAI-compatible embedding client for the Wiki index."""
from __future__ import annotations
import asyncio
import hashlib
import logging
from time import monotonic
import httpx
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
logger = logging.getLogger(__name__)
class WikiEmbeddingError(RuntimeError):
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
super().__init__(message)
self.category = category
self.retryable = retryable
class StrictWikiEmbeddingClient:
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
self.config = config
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
@property
def dimensions(self) -> int | None:
return self._resolved_dimensions
@property
def fingerprint(self) -> str:
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
return []
vectors: list[list[float]] = []
batch_size = max(1, self.config.batch_size)
for start in range(0, len(texts), batch_size):
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
if len(vectors) != len(texts):
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
if self.config.api_key:
headers["Authorization"] = f"Bearer {self.config.api_key}"
payload: dict = {"model": self.config.model, "input": texts}
if self.config.dimensions:
payload["dimensions"] = self.config.dimensions
last_error: WikiEmbeddingError | None = None
for attempt in range(self.config.max_retries + 1):
started = monotonic()
try:
response = await (await self._get_client()).post(url, headers=headers, json=payload)
if response.status_code in {401, 403}:
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
if response.status_code == 429 or response.status_code >= 500:
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
response.raise_for_status()
data = response.json()
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
if len(items) != len(texts):
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
result: list[list[float]] = []
for item in items:
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(vector.size)
result.append(vector.astype(float).tolist())
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
return result
except WikiEmbeddingError as exc:
last_error = exc
except httpx.TimeoutException:
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
except (httpx.HTTPError, ValueError, TypeError) as exc:
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
raise last_error
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
raise last_error or WikiEmbeddingError("Embedding request failed")
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
@@ -0,0 +1,86 @@
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
from __future__ import annotations
import hashlib
import json
import re
from datetime import datetime
from typing import Any
from urllib.parse import unquote
def normalize_slug(value: Any) -> str:
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
slug = re.sub(r"/{2,}", "/", slug)
if not slug:
raise ValueError("Wiki page slug is empty")
return slug
def slug_hash(slug: str) -> str:
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
def _list(value: Any) -> list:
if isinstance(value, list):
return value
if isinstance(value, tuple):
return list(value)
if isinstance(value, str) and value.strip():
return [value.strip()]
return []
def _datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value
raw = str(value or "").strip()
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
return None
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
status = str(remote.get("status") or "published").strip().lower()
if status not in {"draft", "published", "archived"}:
status = "published"
hash_payload = {
"title": title,
"aliases": aliases,
"summary": summary,
"content_md": content,
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
"status": status,
}
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
return {
"knowledge_base_mapping_id": mapping_id,
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
"slug": slug,
"slug_hash": slug_hash(slug),
"title": title,
"summary": summary,
"content_md": content,
"page_type": hash_payload["page_type"],
"status": status,
"aliases": aliases,
"category_path": category_path,
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
"content_hash": content_hash,
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
}
@@ -0,0 +1,248 @@
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
the raw knowledge/chunk search endpoint, so callers always receive processed
Wiki pages even while a whole knowledge base is awaiting vectorization.
"""
from __future__ import annotations
import asyncio
from typing import Any
from urllib.parse import urlencode
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
"""Return whether a completed full scan produced one coherent active index."""
if not state or not fingerprint:
return False
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
return False
if str(state.get("embedding_fingerprint") or "") != fingerprint:
return False
local_count = int(state.get("local_page_count") or 0)
ready_count = int(state.get("ready_page_count") or 0)
failed_count = int(state.get("failed_page_count") or 0)
vector_count = int(state.get("vector_count") or 0)
if failed_count or ready_count != local_count:
return False
return local_count == 0 or vector_count > 0
class WikiRetrievalService:
"""Search complete indexes locally and incomplete indexes through Wiki API."""
def __init__(
self,
store: LlmWikiIndexStore,
*,
vector_search: WikiVectorSearchService | None,
client: WeKnoraClient | None,
) -> None:
self.store = store
self.vector_search = vector_search
self.client = client
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int = 8,
include_drafts: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {
"query": normalized_query,
"mode": "wiki_api",
"results": [],
"knowledge_base_count": 0,
"partial": False,
"warnings": [],
"fallback_knowledge_base_ids": [],
}
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
results: list[dict[str, Any]] = []
warnings: list[str] = []
vector_used = False
fallback_used = False
if vector_mappings and self.vector_search is not None:
try:
payload = await self.vector_search.search(
normalized_query,
vector_mappings,
top_k_pages=max(top_k_pages, len(vector_mappings)),
include_drafts=include_drafts,
)
results.extend(payload.get("results") or [])
warnings.extend(str(item) for item in payload.get("warnings") or [])
vector_used = True
except WikiIndexError as exc:
# A status row can become stale between classification and
# matrix loading. Wiki API remains a safe Wiki-only fallback.
warnings.append(f"local_vector:{exc.code}")
fallback_mappings.extend(vector_mappings)
vector_mappings = []
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
if fallback_mappings:
if self.client is None:
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
else:
batches = await asyncio.gather(
*(
self._search_remote_mapping(
normalized_query,
mapping,
limit=max(top_k_pages, 1),
include_drafts=include_drafts,
)
for mapping in fallback_mappings
),
return_exceptions=True,
)
for mapping, batch in zip(fallback_mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
else:
results.extend(batch)
fallback_used = True
if warnings and not results and not vector_used and not fallback_used:
raise WikiIndexError(
"WIKI_API_UNAVAILABLE",
"Wiki page retrieval is unavailable",
status_code=502 if self.client is not None else 503,
)
# De-duplicate by the public Wiki citation identity. Vector scores and
# Wiki-API rank scores are both descending relevance hints.
unique: dict[tuple[str, str], dict[str, Any]] = {}
for item in results:
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
current = unique.get(key)
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
unique[key] = item
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
return {
"query": normalized_query,
"mode": mode,
"results": ranked[:top_k_pages],
"knowledge_base_count": len(mappings),
"index_fingerprint": fingerprint,
"partial": bool(warnings),
"warnings": warnings,
"fallback_knowledge_base_ids": fallback_ids,
}
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if self.vector_search is None:
return [], list(mappings)
mapping_ids = [str(mapping["id"]) for mapping in mappings]
states = {
str(state["knowledge_base_mapping_id"]): state
for state in await self.store.list_index_status(mapping_ids)
}
fingerprint = self.vector_search.embedding.fingerprint
vector_mappings = [
mapping
for mapping in mappings
if mapping.get("wiki_index_enabled", True)
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
]
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
async def _search_remote_mapping(
self,
query: str,
mapping: dict[str, Any],
*,
limit: int,
include_drafts: bool,
) -> list[dict[str, Any]]:
if self.client is None:
return []
remote_id = str(mapping["weknora_id"])
listing = await self.client.list_wiki_pages(
remote_id,
page=1,
page_size=min(max(limit * 3, 20), 100),
query=query,
)
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
slugs = list(
dict.fromkeys(
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
for item in summaries
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
)
)[: max(limit * 2, limit)]
details = await asyncio.gather(
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
return_exceptions=True,
)
results: list[dict[str, Any]] = []
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
if isinstance(detail, Exception):
continue
try:
page = normalize_wiki_page(str(mapping["id"]), detail)
except (TypeError, ValueError):
continue
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
continue
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
content = str(page.get("content_md") or "")
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
results.append(
{
"kind": "wiki_page",
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
"wiki_slug": slug,
"title": page.get("title") or slug,
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at"),
"score": score,
"content": content,
"snippet": content[:360],
"matched_heading": None,
"matched_sections": [
{
"section_index": 0,
"heading": None,
"content": content,
"score": score,
}
],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
"retrieval_mode": "wiki_api",
}
)
if len(results) >= limit:
break
return results
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
@@ -0,0 +1,48 @@
"""App-to-harness runtime injection for local Wiki index services."""
from __future__ import annotations
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
_store: LlmWikiIndexStore | None = None
_search: WikiVectorSearchService | None = None
_sync: WikiSyncService | None = None
_retrieval: WikiRetrievalService | None = None
def register_local_wiki_index_runtime(
*,
store: LlmWikiIndexStore | None,
search: WikiVectorSearchService | None,
sync: WikiSyncService | None,
retrieval: WikiRetrievalService | None = None,
) -> None:
global _store, _search, _sync, _retrieval
_store, _search, _sync, _retrieval = store, search, sync, retrieval
def get_local_wiki_index_store() -> LlmWikiIndexStore:
if _store is None:
raise RuntimeError("Local Wiki index store is not initialized")
return _store
def get_local_wiki_search_service() -> WikiVectorSearchService:
if _search is None:
raise RuntimeError("Local Wiki vector search is not initialized")
return _search
def get_local_wiki_sync_service() -> WikiSyncService:
if _sync is None:
raise RuntimeError("Local Wiki sync service is not initialized")
return _sync
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
if _retrieval is None:
raise RuntimeError("Wiki-only retrieval service is not initialized")
return _retrieval
@@ -0,0 +1,176 @@
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
from __future__ import annotations
import asyncio
import hashlib
import re
from collections import OrderedDict, defaultdict
from time import monotonic
from typing import Any
from urllib.parse import urlencode
import numpy as np
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
class WikiIndexError(RuntimeError):
def __init__(self, code: str, message: str, *, status_code: int) -> None:
super().__init__(message)
self.code = code
self.status_code = status_code
class WikiVectorSearchService:
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
self.config = config
self.embedding = embedding
self.cache = cache
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
async def search(
self,
query: str,
mappings: list[dict[str, Any]],
*,
top_k_pages: int | None = None,
include_drafts: bool = False,
external: bool = False,
) -> dict[str, Any]:
normalized_query = query.strip()
if not normalized_query:
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
if not mappings:
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
try:
query_vector = await self._embed_query(normalized_query)
except (WikiEmbeddingError, ValueError) as exc:
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
fingerprint = self.embedding.fingerprint
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
batches = await asyncio.gather(*tasks, return_exceptions=True)
candidates: list[dict[str, Any]] = []
warnings: list[str] = []
for mapping, batch in zip(mappings, batches, strict=True):
if isinstance(batch, Exception):
warnings.append(f"{mapping.get('id')}: {batch}")
else:
candidates.extend(batch)
if warnings and not candidates and len(warnings) == len(mappings):
if any("FINGERPRINT" in item for item in warnings):
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
elif any("rebuild" in item.lower() for item in warnings):
code = "WIKI_INDEX_REBUILDING"
else:
code = "WIKI_INDEX_NOT_READY"
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
candidates.sort(key=lambda item: item["score"], reverse=True)
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for candidate in candidates:
page = candidate["page"]
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
if len(grouped[key]) < self.config.max_sections_per_page:
grouped[key].append(candidate)
results: list[dict[str, Any]] = []
for sections in grouped.values():
first = sections[0]
page = first["page"]
mapping = first["mapping"]
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
page_score = first["score"] + (0.05 * second_score if second_score else 0)
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
results.append(
{
"kind": "wiki_page",
"id": page["id"],
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"wiki_page_id": page["id"],
"wiki_slug": page["slug"],
"title": page.get("title") or page["slug"],
"summary": page.get("summary") or "",
"page_type": page.get("page_type") or "article",
"page_version": int(page.get("page_version") or 0),
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
"score": float(page_score),
"content": "\n\n".join(str(item["section_content"]) for item in sections),
"snippet": str(sections[0]["section_content"])[:360],
"matched_heading": sections[0].get("heading"),
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
"url": f"/weknora-source-preview?{params}",
"source": mapping.get("name") or "LLMWiki",
"type": "llmwiki_wiki",
"mode": "citation",
}
)
results.sort(key=lambda item: item["score"], reverse=True)
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
async def _embed_query(self, query: str) -> np.ndarray:
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
key = f"{self.embedding.fingerprint}:{digest}"
cached = self._query_vectors.get(key)
if cached is not None and monotonic() - cached[0] <= 300:
self._query_vectors.move_to_end(key)
return cached[1]
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
key = f"{self.embedding.fingerprint}:{digest}"
self._query_vectors[key] = (monotonic(), vector)
self._query_vectors.move_to_end(key)
while len(self._query_vectors) > 256:
self._query_vectors.popitem(last=False)
return vector
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
mapping_id = str(mapping["id"])
try:
snapshot = await self.cache.get(mapping_id, fingerprint)
except ValueError as exc:
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
if snapshot.matrix.size == 0:
return []
if snapshot.dimensions != vector.size:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
eligible = np.asarray(
[
index
for index, row in enumerate(snapshot.rows)
if row["page"].get("status") != "draft" or (include_drafts and not external)
],
dtype=int,
)
if eligible.size == 0:
return []
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
candidates = []
for index, score in zip(indices, scores, strict=True):
row = snapshot.rows[int(index)]
page = row["page"]
vector_score = float(score)
if vector_score < self.config.min_similarity:
continue
final_score = vector_score
if self.config.keyword_boost_enabled and terms:
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
keyword_score = sum(term in haystack for term in terms) / len(terms)
weight = self.config.keyword_boost_weight
final_score = (1 - weight) * vector_score + weight * keyword_score
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
return candidates
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
scores = snapshot.matrix[eligible] @ vector
count = min(self.config.top_k_sections, scores.size)
if count <= 0:
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
positions = np.argpartition(scores, -count)[-count:]
positions = positions[np.argsort(scores[positions])[::-1]]
return eligible[positions], scores[positions]
@@ -0,0 +1,173 @@
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
from __future__ import annotations
import asyncio
import logging
import platform
from typing import Any
from uuid import uuid4
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
logger = logging.getLogger(__name__)
class WikiSyncLeaseLost(RuntimeError):
pass
def _safe_error(exc: BaseException) -> str:
return f"{type(exc).__name__}: {str(exc)[:1000]}"
class WikiSyncService:
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
self.config = config
self.store = store
self.embedding = embedding
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
# Auto-dimension configurations need one successful response to
# produce a stable fingerprint before lease/rebuild decisions.
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
fingerprint = self.embedding.fingerprint
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
await self.store.mark_rebuild_required([mapping_id], fingerprint)
self.cache.invalidate(mapping_id)
prior_state = "rebuild_required"
rebuilding = prior_state == "rebuild_required"
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
if not acquired:
return {"mapping_id": mapping_id, "status": "already_running"}
lease_lost = asyncio.Event()
async def _lease_heartbeat() -> None:
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
while True:
await asyncio.sleep(interval)
try:
renewed = await self.store.renew_sync_lease(
mapping_id,
owner=self.owner,
lease_seconds=self.config.sync_lease_seconds,
)
except Exception:
renewed = False
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
if not renewed:
lease_lost.set()
return
heartbeat = asyncio.create_task(_lease_heartbeat())
def _ensure_lease() -> None:
if lease_lost.is_set():
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
remote_count = 0
changed = 0
failed = 0
try:
remote_id = str(mapping["weknora_id"])
await self.client.get_knowledge_base(remote_id)
page_number = 1
while True:
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
remote_count += len(pages)
for summary in pages:
try:
_ensure_lease()
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
remote_page = await self.client.get_wiki_page(remote_id, slug)
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
active_fingerprint = self.embedding.fingerprint
if (
not force
and existing
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
continue
sections = (
[]
if page["status"] == "archived"
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
)
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
fingerprint = self.embedding.fingerprint
vectors = []
for section, vector in zip(sections, embeddings, strict=True):
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
vectors.append(
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
)
_ensure_lease()
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
self.cache.invalidate(mapping_id)
changed += 1
except Exception as exc:
if isinstance(exc, WikiSyncLeaseLost):
raise
failed += 1
try:
fallback = normalize_wiki_page(mapping_id, summary)
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
except Exception as record_exc:
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
_ensure_lease()
total_pages = int(batch.get("total_pages") or 1)
if page_number >= total_pages:
break
page_number += 1
_ensure_lease()
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
if deleted:
self.cache.invalidate(mapping_id)
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=not (rebuilding and failed),
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
owner=self.owner,
sync_id=sync_id,
success=False,
remote_page_count=remote_count,
error="Sync cancelled",
)
raise
except Exception as exc:
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
raise
finally:
heartbeat.cancel()
await asyncio.gather(heartbeat, return_exceptions=True)
@@ -0,0 +1,86 @@
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from time import monotonic
from typing import Any
import numpy as np
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
@dataclass(frozen=True)
class WikiVectorSnapshot:
mapping_id: str
index_revision: int
embedding_fingerprint: str
dimensions: int
matrix: np.ndarray
rows: list[dict[str, Any]]
class WikiVectorCache:
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
self.store = store
self.max_knowledge_bases = max_knowledge_bases
self.revision_check_seconds = revision_check_seconds
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
self._checked_at: dict[str, float] = {}
self._locks: dict[str, asyncio.Lock] = {}
def invalidate(self, mapping_id: str) -> None:
self._snapshots.pop(mapping_id, None)
self._checked_at.pop(mapping_id, None)
def clear(self) -> None:
self._snapshots.clear()
self._checked_at.clear()
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
cached = self._snapshots.get(mapping_id)
now = monotonic()
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
self._snapshots.move_to_end(mapping_id)
return cached
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
async with lock:
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
self._checked_at[mapping_id] = monotonic()
cached = self._snapshots.get(mapping_id)
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
if active_fingerprint != fingerprint:
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
self._snapshots.move_to_end(mapping_id)
return cached
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
self._snapshots[mapping_id] = snapshot
self._snapshots.move_to_end(mapping_id)
while len(self._snapshots) > self.max_knowledge_bases:
evicted, _ = self._snapshots.popitem(last=False)
self._checked_at.pop(evicted, None)
return snapshot
@staticmethod
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
rows = list(raw.get("rows") or [])
if not rows:
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
dimensions = int(rows[0]["embedding_dimensions"])
vectors = []
valid_rows = []
for row in rows:
if int(row["embedding_dimensions"]) != dimensions:
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
vectors.append(decode_vector(row["vector_blob"], dimensions))
valid_rows.append(row)
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
matrix.setflags(write=False)
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
@@ -0,0 +1,33 @@
"""Validated little-endian normalized float32 Wiki vector codec."""
from __future__ import annotations
import numpy as np
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
vector = np.asarray(values, dtype=np.float32)
if vector.ndim != 1 or vector.size == 0:
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
if dimensions is not None and vector.size != dimensions:
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
if not np.isfinite(vector).all():
raise ValueError("Embedding vector contains NaN or infinity")
norm = float(np.linalg.norm(vector))
if not np.isfinite(norm) or norm <= 0:
raise ValueError("Embedding vector has zero norm")
return np.asarray(vector / norm, dtype="<f4")
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
vector = normalize_vector(values, dimensions=dimensions)
return vector.tobytes(order="C"), int(vector.size)
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
if len(blob) != dimensions * 4:
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
vector = np.frombuffer(blob, dtype="<f4")
if not np.isfinite(vector).all():
raise ValueError("Stored vector contains NaN or infinity")
return vector.copy()
@@ -22,7 +22,15 @@ class LlmWikiStore(ABC):
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def adopt_conversation_deposit_mapping(
@@ -33,6 +33,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
"published_by": None,
"reviewed_by": None,
"published_at": None,
"wiki_index_enabled": True,
"external_search_enabled": False,
"external_search_updated_at": None,
"created_at": now,
"updated_at": now,
}
@@ -65,7 +68,15 @@ class MemoryLlmWikiStore(LlmWikiStore):
return dict(row)
return None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
@@ -73,6 +84,11 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
row["updated_at"] = _now()
return dict(row)
@@ -143,11 +159,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if (
row.get("owner_user_id") == "system"
and row.get("name") == "对话沉淀"
and row.get("publication_status") == "published"
):
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,9 @@ class LlmWikiKnowledgeBaseRow(Base):
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -19,7 +19,7 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("published_at", "created_at", "updated_at"):
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
@@ -82,7 +82,15 @@ class LlmWikiRepository(LlmWikiStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
@@ -91,6 +99,11 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
@@ -0,0 +1,21 @@
"""Persistence facade for the DeerFlow-local Wiki vector index."""
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory):
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
__all__ = [
"LlmWikiIndexStore",
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
]
@@ -0,0 +1,54 @@
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
@abstractmethod
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@abstractmethod
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
@abstractmethod
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
@@ -0,0 +1,160 @@
"""In-memory local Wiki index store used by tests and DB-less mode."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
def _now() -> datetime:
return datetime.now(UTC)
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return dict(row) if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
row = {
**(existing or {}),
**page,
"id": page_id,
"index_status": "ready",
"indexed_fingerprint": fingerprint,
"indexed_at": now,
"last_error": None,
"is_remote_deleted": False,
"last_seen_sync_id": sync_id,
"created_at": (existing or {}).get("created_at", now),
"updated_at": now,
}
self.pages[page_id] = row
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
state = self._state(page["knowledge_base_mapping_id"])
state["index_revision"] += 1
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
{
"last_seen_sync_id": sync_id,
"last_error": error,
# Preserve the last atomically completed generation.
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
page.update({"is_remote_deleted": True, "index_status": "deleted"})
self.vectors.pop(page["id"], None)
changed += 1
if changed:
self._state(mapping_id)["index_revision"] += 1
return changed
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
rows = []
for page_id, vectors in self.vectors.items():
page = self.pages[page_id]
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
continue
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
state = self._state(mapping_id)
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
state = self._state(mapping_id)
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
state = self._state(mapping_id)
now = _now()
expires = state.get("lease_expires_at")
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
return False
state.update(
{
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
"active_sync_id": sync_id,
"lease_owner": owner,
"lease_expires_at": now + timedelta(seconds=lease_seconds),
"last_started_at": now,
}
)
return True
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner:
return False
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
{
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
"lease_owner": None,
"lease_expires_at": None,
"active_sync_id": None,
"last_error": error,
"remote_page_count": remote_page_count,
"local_page_count": len(pages),
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
}
)
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
for mapping_id in mapping_ids:
state = self._state(mapping_id)
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
page["index_status"] = "stale"
return len(mapping_ids)
@@ -0,0 +1,94 @@
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class LlmWikiWikiPageRow(Base):
__tablename__ = "llmwiki_wiki_pages"
__table_args__ = (
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
class LlmWikiWikiVectorRow(Base):
__tablename__ = "llmwiki_wiki_vectors"
__table_args__ = (
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -0,0 +1,295 @@
"""SQLAlchemy implementation of the local Wiki index store."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row) -> dict[str, Any]:
value = row.to_dict()
for key, item in tuple(value.items()):
if isinstance(item, datetime):
value[key] = item.isoformat()
return value
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if row is None:
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
session.add(row)
await session.flush()
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async with self._sf() as session:
await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
else:
for key, value in page.items():
setattr(row, key, value)
row.index_status = "ready"
row.indexed_fingerprint = fingerprint
row.indexed_at = now
row.last_error = None
row.is_remote_deleted = False
row.last_seen_sync_id = sync_id
row.updated_at = now
await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
session.add_all(
[
LlmWikiWikiVectorRow(
id=str(uuid4()),
page_id=row.id,
knowledge_base_mapping_id=mapping_id,
section_index=int(item["section_index"]),
heading=item.get("heading"),
section_content=str(item["section_content"]),
content_hash=str(item["content_hash"]),
embedding_fingerprint=fingerprint,
embedding_dimensions=int(item["embedding_dimensions"]),
vector_blob=bytes(item["vector_blob"]),
created_at=now,
)
for item in vectors
]
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
row.index_status = "failed"
else:
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
# Preserve the last atomically completed generation while the
# newer remote version remains pending after an embedding error.
row.index_status = "ready" if vector_count else "failed"
row.last_seen_sync_id = sync_id
row.last_error = safe_error
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async with self._sf() as session:
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
)
.with_for_update()
)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return 0
now = _now()
page_ids = [row.id for row in rows]
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
for row in rows:
row.is_remote_deleted = True
row.index_status = "deleted"
row.updated_at = now
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return len(rows)
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
)
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None:
return 0, None, "not_synced"
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
now = _now()
async with self._sf() as session:
try:
await self._ensure_state(session, mapping_id)
await session.commit()
except IntegrityError:
# Another worker created the one-per-KB state row between our
# read and insert. Roll back this isolated bootstrap session;
# the conditional UPDATE below is the actual lease election.
await session.rollback()
async with self._sf() as session:
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
or_(
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at < now,
),
)
.values(
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
active_sync_id=sync_id,
lease_owner=owner,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_started_at=now,
last_error=None,
updated_at=now,
)
)
result = await session.execute(stmt)
await session.commit()
return bool(result.rowcount)
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(LlmWikiWikiSyncStateRow)
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
failed_count = int(
(
await session.execute(
select(func.count())
.select_from(LlmWikiWikiPageRow)
.where(
*visible,
or_(
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
LlmWikiWikiPageRow.last_error.is_not(None),
),
)
)
).scalar_one()
)
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
now = _now()
was_rebuilding = state.state == "rebuilding"
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
state.active_sync_id = None
state.lease_owner = None
state.lease_expires_at = None
state.last_error = (error or "")[:4000] or None
state.remote_page_count = remote_page_count
state.local_page_count = local_count
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)
if mapping_ids:
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
if not mapping_ids:
return 0
now = _now()
async with self._sf() as session:
for mapping_id in mapping_ids:
state = await self._ensure_state(session, mapping_id)
state.state = "rebuild_required"
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
await session.commit()
return len(mapping_ids)
@@ -0,0 +1,143 @@
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
Revision ID: 20260823_01
Revises: 20260821_02
Create Date: 2026-08-23
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mysql
revision = "20260823_01"
down_revision = "20260821_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
if "wiki_index_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
if "external_search_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
if "external_search_updated_at" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
if not inspector.has_table("llmwiki_wiki_pages"):
op.create_table(
"llmwiki_wiki_pages",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
sa.Column("remote_page_id", sa.String(128), nullable=True),
sa.Column("slug", sa.String(1024), nullable=False),
sa.Column("slug_hash", sa.String(64), nullable=False),
sa.Column("title", sa.String(512), nullable=False, server_default=""),
sa.Column("summary", long_text, nullable=False),
sa.Column("content_md", long_text, nullable=False),
sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
sa.Column("status", sa.String(32), nullable=False, server_default="published"),
sa.Column("aliases", portable_json, nullable=False),
sa.Column("category_path", portable_json, nullable=False),
sa.Column("parent_slug", sa.String(1024), nullable=True),
sa.Column("wiki_path", sa.String(1024), nullable=True),
sa.Column("source_refs", portable_json, nullable=False),
sa.Column("chunk_refs", portable_json, nullable=False),
sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
)
op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
if not inspector.has_table("llmwiki_wiki_vectors"):
op.create_table(
"llmwiki_wiki_vectors",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
sa.Column("section_index", sa.Integer(), nullable=False),
sa.Column("heading", sa.String(1024), nullable=True),
sa.Column("section_content", long_text, nullable=False),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
)
op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
if not inspector.has_table("llmwiki_wiki_sync_states"):
op.create_table(
"llmwiki_wiki_sync_states",
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
sa.Column("active_sync_id", sa.String(36), nullable=True),
sa.Column("lease_owner", sa.String(128), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
if inspector.has_table(table):
op.drop_table(table)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
if index in indexes:
op.drop_index(index, table_name="llmwiki_knowledge_bases")
for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
if column in columns:
op.drop_column("llmwiki_knowledge_bases", column)
@@ -52,6 +52,7 @@ from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.models.run_event import RunEventRow
@@ -129,6 +130,9 @@ __all__ = [
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiWikiPageRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"NotificationRow",
"ParallelAgentTaskRow",
@@ -6,28 +6,25 @@ import json
import re
from collections.abc import Mapping
from typing import Any
from urllib.parse import urlencode
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
_CONVERSATION_DEPOSIT_NAME = "对话沉淀"
_CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
return (
str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
)
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
def _sanitize_conversation_deposit_value(value: Any) -> Any:
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
query: Focused natural-language retrieval query.
"""
local_config = get_app_config().llmwiki.local_wiki_index
resolved = get_resolved_llmwiki_runtime()
if not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
store = get_llmwiki_store()
user_id = get_effective_user_id()
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
return "No authorized LLMWiki knowledge bases are available."
try:
results = await build_weknora_client(resolved).search(
query.strip(),
[str(row["weknora_id"]) for row in rows],
)
except (RuntimeError, WeKnoraError) as exc:
return f"LLMWiki retrieval failed: {exc}"
local_by_remote = {str(row["weknora_id"]): row for row in rows}
citations = []
for result in results:
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
visible_result = (
_sanitize_conversation_deposit_value(result)
if _is_conversation_deposit_mapping(mapping)
else result
)
params = urlencode(
{
"knowledge_base_id": mapping["id"],
"chunk_id": visible_result.get("chunk_id") or "",
"knowledge_id": visible_result.get("knowledge_id") or "",
"document_id": visible_result.get("knowledge_id") or "",
"wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
}
)
citations.append(
{
"title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
"content": visible_result.get("content") or "",
"score": visible_result.get("score") or 0,
"knowledge_base_id": mapping["id"],
"knowledge_base_name": mapping.get("name") or "",
"source": mapping.get("name") or "LLMWiki",
"knowledge_id": visible_result.get("knowledge_id") or "",
"chunk_id": visible_result.get("chunk_id") or "",
"url": f"/weknora-source-preview?{params}",
}
)
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
except (RuntimeError, WikiIndexError) as exc:
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
mapping_by_id = {str(row["id"]): row for row in rows}
results = []
for item in payload["results"]:
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
results.append(visible)
return json.dumps(
{
"query": query,
"selection": [row["id"] for row in rows],
"knowledge_base_count": len(rows),
"results": citations,
"mode": payload.get("mode") or "wiki_api",
"results": results,
"partial": payload.get("partial", False),
"warnings": payload.get("warnings", []),
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
},
ensure_ascii=False,
)