feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
+100
-9
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def normalize_service_url(value: str | None) -> str:
|
||||
@@ -37,18 +37,12 @@ class WeKnoraConnectionConfig(BaseModel):
|
||||
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
|
||||
admin_email: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"Optional WeKnora administrator email used by DeerFlow's server-side "
|
||||
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
|
||||
),
|
||||
description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
||||
)
|
||||
admin_password: str = Field(
|
||||
default="",
|
||||
repr=False,
|
||||
description=(
|
||||
"Optional WeKnora administrator password used by DeerFlow's server-side "
|
||||
"WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
|
||||
),
|
||||
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@@ -58,10 +52,107 @@ class WeKnoraConnectionConfig(BaseModel):
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class LocalWikiEmbeddingConfig(BaseModel):
|
||||
enabled: bool = True
|
||||
provider: str = "openai"
|
||||
model: str = "bge-m3"
|
||||
base_url: str = ""
|
||||
api_key: str = Field(default="", repr=False)
|
||||
dimensions: int | None = Field(default=1024, ge=1, le=65536)
|
||||
batch_size: int = Field(default=32, ge=1, le=512)
|
||||
timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
max_retries: int = Field(default=3, ge=0, le=10)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@field_validator("provider")
|
||||
@classmethod
|
||||
def _validate_provider(cls, value: str) -> str:
|
||||
normalized = value.strip().lower()
|
||||
if normalized not in {"openai", "openai_compatible"}:
|
||||
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
|
||||
return normalized
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def _validate_url(cls, value: str) -> str:
|
||||
return normalize_service_url(value)
|
||||
|
||||
|
||||
class LocalWikiExternalApiConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
api_key: str = Field(default="", repr=False)
|
||||
previous_api_key: str = Field(default="", repr=False)
|
||||
max_query_chars: int = Field(default=1000, ge=1, le=10000)
|
||||
max_knowledge_bases: int = Field(default=20, ge=1, le=100)
|
||||
max_results: int = Field(default=50, ge=1, le=100)
|
||||
default_results: int = Field(default=10, ge=1, le=100)
|
||||
include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
|
||||
requests_per_minute: int = Field(default=60, ge=1, le=100000)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_enabled_key(self):
|
||||
if self.enabled and not self.api_key.strip():
|
||||
raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
|
||||
if self.default_results > self.max_results:
|
||||
raise ValueError("external_api.default_results must not exceed max_results")
|
||||
return self
|
||||
|
||||
|
||||
class LocalWikiIndexConfig(BaseModel):
|
||||
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
||||
|
||||
enabled: bool = False
|
||||
sync_enabled: bool = False
|
||||
internal_search_enabled: bool = False
|
||||
reference_ui_enabled: bool = False
|
||||
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
|
||||
sync_page_size: int = Field(default=100, ge=1, le=500)
|
||||
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
||||
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
|
||||
sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
|
||||
sync_max_retries: int = Field(default=3, ge=0, le=10)
|
||||
sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
|
||||
chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
|
||||
chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
|
||||
chunk_min_chars: int = Field(default=80, ge=1, le=10000)
|
||||
title_prefix_enabled: bool = True
|
||||
embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
|
||||
engine: str = "numpy_exact"
|
||||
strict_vector: bool = True
|
||||
top_k_sections: int = Field(default=30, ge=1, le=1000)
|
||||
top_k_pages: int = Field(default=8, ge=1, le=100)
|
||||
max_sections_per_page: int = Field(default=2, ge=1, le=20)
|
||||
min_similarity: float = Field(default=0.35, ge=-1, le=1)
|
||||
keyword_boost_enabled: bool = False
|
||||
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
||||
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
||||
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
||||
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_index(self):
|
||||
if self.chunk_overlap_chars >= self.chunk_max_chars:
|
||||
raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
|
||||
if self.chunk_min_chars > self.chunk_max_chars:
|
||||
raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
|
||||
if self.top_k_pages > self.top_k_sections:
|
||||
raise ValueError("top_k_pages must not exceed top_k_sections")
|
||||
if self.engine != "numpy_exact":
|
||||
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
|
||||
if self.enabled and not self.embedding.enabled:
|
||||
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
|
||||
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
|
||||
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
|
||||
return self
|
||||
|
||||
|
||||
class LlmWikiConfig(BaseModel):
|
||||
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
|
||||
|
||||
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
|
||||
local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
|
||||
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""Markdown-aware Wiki section chunking."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
|
||||
_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
|
||||
_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
|
||||
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiSection:
|
||||
section_index: int
|
||||
heading: str | None
|
||||
content: str
|
||||
embedding_text: str
|
||||
content_hash: str
|
||||
|
||||
|
||||
def _clean_markdown(value: str) -> str:
|
||||
text = value.replace("\r\n", "\n")
|
||||
text = _FRONTMATTER_RE.sub("", text)
|
||||
text = _SCRIPT_STYLE_RE.sub("", text)
|
||||
text = _IMAGE_RE.sub(lambda match: match.group(1), text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
|
||||
paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
|
||||
chunks: list[str] = []
|
||||
buffer = ""
|
||||
for paragraph in paragraphs:
|
||||
if len(buffer) + len(paragraph) + 2 <= max_chars:
|
||||
buffer = f"{buffer}\n\n{paragraph}".strip()
|
||||
continue
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
buffer = ""
|
||||
if len(paragraph) <= max_chars:
|
||||
buffer = paragraph
|
||||
continue
|
||||
step = max(1, max_chars - overlap)
|
||||
chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
|
||||
if buffer:
|
||||
chunks.append(buffer)
|
||||
return chunks
|
||||
|
||||
|
||||
def chunk_wiki_page(
|
||||
page: dict,
|
||||
*,
|
||||
max_chars: int,
|
||||
overlap: int,
|
||||
min_chars: int,
|
||||
title_prefix_enabled: bool,
|
||||
) -> list[WikiSection]:
|
||||
content = _clean_markdown(str(page.get("content_md") or ""))
|
||||
heading_stack: list[str] = []
|
||||
sections: list[tuple[str | None, str]] = []
|
||||
current_heading: str | None = None
|
||||
body: list[str] = []
|
||||
for line in content.splitlines():
|
||||
match = _HEADING_RE.match(line)
|
||||
if not match:
|
||||
body.append(line)
|
||||
continue
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
level = len(match.group(1))
|
||||
heading_stack = heading_stack[: level - 1]
|
||||
heading_stack.append(match.group(2).strip())
|
||||
current_heading = " / ".join(heading_stack)
|
||||
body = []
|
||||
if any(item.strip() for item in body):
|
||||
sections.append((current_heading, "\n".join(body).strip()))
|
||||
if content and not sections:
|
||||
sections.append((None, content))
|
||||
summary = _clean_markdown(str(page.get("summary") or ""))
|
||||
if summary:
|
||||
sections.insert(0, ("页面摘要", summary))
|
||||
|
||||
prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
|
||||
aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
|
||||
if aliases:
|
||||
prefix_lines.append(f"别名:{'、'.join(aliases)}")
|
||||
if page.get("page_type"):
|
||||
prefix_lines.append(f"类型:{page['page_type']}")
|
||||
seen: set[str] = set()
|
||||
result: list[WikiSection] = []
|
||||
for heading, section_text in sections:
|
||||
for chunk in _windows(section_text, max_chars, overlap):
|
||||
normalized = re.sub(r"\s+", " ", chunk).strip()
|
||||
if len(normalized) < min_chars and len(sections) > 1:
|
||||
continue
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
if digest in seen:
|
||||
continue
|
||||
seen.add(digest)
|
||||
context = list(prefix_lines) if title_prefix_enabled else []
|
||||
if heading:
|
||||
context.append(f"章节:{heading}")
|
||||
embedding_text = "\n".join([*context, "", chunk]).strip()
|
||||
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
|
||||
return result
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""Strict OpenAI-compatible embedding client for the Wiki index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from time import monotonic
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WikiEmbeddingError(RuntimeError):
|
||||
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
||||
super().__init__(message)
|
||||
self.category = category
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
class StrictWikiEmbeddingClient:
|
||||
def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
|
||||
self.config = config
|
||||
self._client = http_client
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
return self._resolved_dimensions
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._owns_client and self._client is not None:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None:
|
||||
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
|
||||
return self._client
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
return []
|
||||
vectors: list[list[float]] = []
|
||||
batch_size = max(1, self.config.batch_size)
|
||||
for start in range(0, len(texts), batch_size):
|
||||
vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
|
||||
if len(vectors) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
|
||||
return vectors
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
payload: dict = {"model": self.config.model, "input": texts}
|
||||
if self.config.dimensions:
|
||||
payload["dimensions"] = self.config.dimensions
|
||||
last_error: WikiEmbeddingError | None = None
|
||||
for attempt in range(self.config.max_retries + 1):
|
||||
started = monotonic()
|
||||
try:
|
||||
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
||||
if response.status_code in {401, 403}:
|
||||
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
|
||||
if response.status_code == 429 or response.status_code >= 500:
|
||||
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
||||
if len(items) != len(texts):
|
||||
raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
|
||||
result: list[list[float]] = []
|
||||
for item in items:
|
||||
vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(vector.size)
|
||||
result.append(vector.astype(float).tolist())
|
||||
logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
|
||||
return result
|
||||
except WikiEmbeddingError as exc:
|
||||
last_error = exc
|
||||
except httpx.TimeoutException:
|
||||
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
||||
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
||||
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
|
||||
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
||||
raise last_error
|
||||
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
||||
raise last_error or WikiEmbeddingError("Embedding request failed")
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
result = await self.embed_texts([text])
|
||||
return result[0]
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
def normalize_slug(value: Any) -> str:
|
||||
slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
|
||||
slug = re.sub(r"/{2,}", "/", slug)
|
||||
if not slug:
|
||||
raise ValueError("Wiki page slug is empty")
|
||||
return slug
|
||||
|
||||
|
||||
def slug_hash(slug: str) -> str:
|
||||
return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _list(value: Any) -> list:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return [value.strip()]
|
||||
return []
|
||||
|
||||
|
||||
def _datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
|
||||
slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
|
||||
aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
|
||||
category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
|
||||
title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
|
||||
summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
|
||||
content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
|
||||
status = str(remote.get("status") or "published").strip().lower()
|
||||
if status not in {"draft", "published", "archived"}:
|
||||
status = "published"
|
||||
hash_payload = {
|
||||
"title": title,
|
||||
"aliases": aliases,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
|
||||
"status": status,
|
||||
}
|
||||
content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
||||
return {
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
|
||||
"slug": slug,
|
||||
"slug_hash": slug_hash(slug),
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"content_md": content,
|
||||
"page_type": hash_payload["page_type"],
|
||||
"status": status,
|
||||
"aliases": aliases,
|
||||
"category_path": category_path,
|
||||
"parent_slug": str(remote.get("parent_slug") or "").strip() or None,
|
||||
"wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
|
||||
"source_refs": _list(remote.get("source_refs") or remote.get("sources")),
|
||||
"chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
|
||||
"page_version": int(remote.get("page_version") or remote.get("version") or 0),
|
||||
"content_hash": content_hash,
|
||||
"remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
|
||||
"remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
|
||||
|
||||
The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
|
||||
the raw knowledge/chunk search endpoint, so callers always receive processed
|
||||
Wiki pages even while a whole knowledge base is awaiting vectorization.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
|
||||
"""Return whether a completed full scan produced one coherent active index."""
|
||||
|
||||
if not state or not fingerprint:
|
||||
return False
|
||||
if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
|
||||
return False
|
||||
if str(state.get("embedding_fingerprint") or "") != fingerprint:
|
||||
return False
|
||||
local_count = int(state.get("local_page_count") or 0)
|
||||
ready_count = int(state.get("ready_page_count") or 0)
|
||||
failed_count = int(state.get("failed_page_count") or 0)
|
||||
vector_count = int(state.get("vector_count") or 0)
|
||||
if failed_count or ready_count != local_count:
|
||||
return False
|
||||
return local_count == 0 or vector_count > 0
|
||||
|
||||
|
||||
class WikiRetrievalService:
|
||||
"""Search complete indexes locally and incomplete indexes through Wiki API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: LlmWikiIndexStore,
|
||||
*,
|
||||
vector_search: WikiVectorSearchService | None,
|
||||
client: WeKnoraClient | None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.vector_search = vector_search
|
||||
self.client = client
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int = 8,
|
||||
include_drafts: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": "wiki_api",
|
||||
"results": [],
|
||||
"knowledge_base_count": 0,
|
||||
"partial": False,
|
||||
"warnings": [],
|
||||
"fallback_knowledge_base_ids": [],
|
||||
}
|
||||
|
||||
vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
|
||||
results: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
vector_used = False
|
||||
fallback_used = False
|
||||
|
||||
if vector_mappings and self.vector_search is not None:
|
||||
try:
|
||||
payload = await self.vector_search.search(
|
||||
normalized_query,
|
||||
vector_mappings,
|
||||
top_k_pages=max(top_k_pages, len(vector_mappings)),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
results.extend(payload.get("results") or [])
|
||||
warnings.extend(str(item) for item in payload.get("warnings") or [])
|
||||
vector_used = True
|
||||
except WikiIndexError as exc:
|
||||
# A status row can become stale between classification and
|
||||
# matrix loading. Wiki API remains a safe Wiki-only fallback.
|
||||
warnings.append(f"local_vector:{exc.code}")
|
||||
fallback_mappings.extend(vector_mappings)
|
||||
vector_mappings = []
|
||||
|
||||
fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
|
||||
if fallback_mappings:
|
||||
if self.client is None:
|
||||
warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
|
||||
else:
|
||||
batches = await asyncio.gather(
|
||||
*(
|
||||
self._search_remote_mapping(
|
||||
normalized_query,
|
||||
mapping,
|
||||
limit=max(top_k_pages, 1),
|
||||
include_drafts=include_drafts,
|
||||
)
|
||||
for mapping in fallback_mappings
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for mapping, batch in zip(fallback_mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
|
||||
else:
|
||||
results.extend(batch)
|
||||
fallback_used = True
|
||||
|
||||
if warnings and not results and not vector_used and not fallback_used:
|
||||
raise WikiIndexError(
|
||||
"WIKI_API_UNAVAILABLE",
|
||||
"Wiki page retrieval is unavailable",
|
||||
status_code=502 if self.client is not None else 503,
|
||||
)
|
||||
|
||||
# De-duplicate by the public Wiki citation identity. Vector scores and
|
||||
# Wiki-API rank scores are both descending relevance hints.
|
||||
unique: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for item in results:
|
||||
key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
|
||||
current = unique.get(key)
|
||||
if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
|
||||
unique[key] = item
|
||||
ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
|
||||
mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
|
||||
fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
|
||||
return {
|
||||
"query": normalized_query,
|
||||
"mode": mode,
|
||||
"results": ranked[:top_k_pages],
|
||||
"knowledge_base_count": len(mappings),
|
||||
"index_fingerprint": fingerprint,
|
||||
"partial": bool(warnings),
|
||||
"warnings": warnings,
|
||||
"fallback_knowledge_base_ids": fallback_ids,
|
||||
}
|
||||
|
||||
async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if self.vector_search is None:
|
||||
return [], list(mappings)
|
||||
mapping_ids = [str(mapping["id"]) for mapping in mappings]
|
||||
states = {
|
||||
str(state["knowledge_base_mapping_id"]): state
|
||||
for state in await self.store.list_index_status(mapping_ids)
|
||||
}
|
||||
fingerprint = self.vector_search.embedding.fingerprint
|
||||
vector_mappings = [
|
||||
mapping
|
||||
for mapping in mappings
|
||||
if mapping.get("wiki_index_enabled", True)
|
||||
and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
|
||||
]
|
||||
ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
|
||||
return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
|
||||
|
||||
async def _search_remote_mapping(
|
||||
self,
|
||||
query: str,
|
||||
mapping: dict[str, Any],
|
||||
*,
|
||||
limit: int,
|
||||
include_drafts: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if self.client is None:
|
||||
return []
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
listing = await self.client.list_wiki_pages(
|
||||
remote_id,
|
||||
page=1,
|
||||
page_size=min(max(limit * 3, 20), 100),
|
||||
query=query,
|
||||
)
|
||||
summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
|
||||
slugs = list(
|
||||
dict.fromkeys(
|
||||
str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
for item in summaries
|
||||
if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
|
||||
)
|
||||
)[: max(limit * 2, limit)]
|
||||
details = await asyncio.gather(
|
||||
*(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
results: list[dict[str, Any]] = []
|
||||
for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
|
||||
if isinstance(detail, Exception):
|
||||
continue
|
||||
try:
|
||||
page = normalize_wiki_page(str(mapping["id"]), detail)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
|
||||
continue
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
|
||||
content = str(page.get("content_md") or "")
|
||||
score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
|
||||
"wiki_slug": slug,
|
||||
"title": page.get("title") or slug,
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at"),
|
||||
"score": score,
|
||||
"content": content,
|
||||
"snippet": content[:360],
|
||||
"matched_heading": None,
|
||||
"matched_sections": [
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"content": content,
|
||||
"score": score,
|
||||
}
|
||||
],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
"retrieval_mode": "wiki_api",
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""App-to-harness runtime injection for local Wiki index services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
_store: LlmWikiIndexStore | None = None
|
||||
_search: WikiVectorSearchService | None = None
|
||||
_sync: WikiSyncService | None = None
|
||||
_retrieval: WikiRetrievalService | None = None
|
||||
|
||||
|
||||
def register_local_wiki_index_runtime(
|
||||
*,
|
||||
store: LlmWikiIndexStore | None,
|
||||
search: WikiVectorSearchService | None,
|
||||
sync: WikiSyncService | None,
|
||||
retrieval: WikiRetrievalService | None = None,
|
||||
) -> None:
|
||||
global _store, _search, _sync, _retrieval
|
||||
_store, _search, _sync, _retrieval = store, search, sync, retrieval
|
||||
|
||||
|
||||
def get_local_wiki_index_store() -> LlmWikiIndexStore:
|
||||
if _store is None:
|
||||
raise RuntimeError("Local Wiki index store is not initialized")
|
||||
return _store
|
||||
|
||||
|
||||
def get_local_wiki_search_service() -> WikiVectorSearchService:
|
||||
if _search is None:
|
||||
raise RuntimeError("Local Wiki vector search is not initialized")
|
||||
return _search
|
||||
|
||||
|
||||
def get_local_wiki_sync_service() -> WikiSyncService:
|
||||
if _sync is None:
|
||||
raise RuntimeError("Local Wiki sync service is not initialized")
|
||||
return _sync
|
||||
|
||||
|
||||
def get_local_wiki_retrieval_service() -> WikiRetrievalService:
|
||||
if _retrieval is None:
|
||||
raise RuntimeError("Wiki-only retrieval service is not initialized")
|
||||
return _retrieval
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
from collections import OrderedDict, defaultdict
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
|
||||
|
||||
|
||||
class WikiIndexError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, *, status_code: int) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class WikiVectorSearchService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
|
||||
self.config = config
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
mappings: list[dict[str, Any]],
|
||||
*,
|
||||
top_k_pages: int | None = None,
|
||||
include_drafts: bool = False,
|
||||
external: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
|
||||
if not mappings:
|
||||
return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
|
||||
try:
|
||||
query_vector = await self._embed_query(normalized_query)
|
||||
except (WikiEmbeddingError, ValueError) as exc:
|
||||
raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
|
||||
fingerprint = self.embedding.fingerprint
|
||||
tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
|
||||
batches = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
candidates: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
for mapping, batch in zip(mappings, batches, strict=True):
|
||||
if isinstance(batch, Exception):
|
||||
warnings.append(f"{mapping.get('id')}: {batch}")
|
||||
else:
|
||||
candidates.extend(batch)
|
||||
if warnings and not candidates and len(warnings) == len(mappings):
|
||||
if any("FINGERPRINT" in item for item in warnings):
|
||||
code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
|
||||
elif any("rebuild" in item.lower() for item in warnings):
|
||||
code = "WIKI_INDEX_REBUILDING"
|
||||
else:
|
||||
code = "WIKI_INDEX_NOT_READY"
|
||||
raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
|
||||
candidates.sort(key=lambda item: item["score"], reverse=True)
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for candidate in candidates:
|
||||
page = candidate["page"]
|
||||
key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
|
||||
if len(grouped[key]) < self.config.max_sections_per_page:
|
||||
grouped[key].append(candidate)
|
||||
results: list[dict[str, Any]] = []
|
||||
for sections in grouped.values():
|
||||
first = sections[0]
|
||||
page = first["page"]
|
||||
mapping = first["mapping"]
|
||||
second_score = sections[1]["score"] if len(sections) > 1 else 0.0
|
||||
page_score = first["score"] + (0.05 * second_score if second_score else 0)
|
||||
params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
|
||||
results.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"id": page["id"],
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"wiki_page_id": page["id"],
|
||||
"wiki_slug": page["slug"],
|
||||
"title": page.get("title") or page["slug"],
|
||||
"summary": page.get("summary") or "",
|
||||
"page_type": page.get("page_type") or "article",
|
||||
"page_version": int(page.get("page_version") or 0),
|
||||
"updated_at": page.get("remote_updated_at") or page.get("updated_at"),
|
||||
"score": float(page_score),
|
||||
"content": "\n\n".join(str(item["section_content"]) for item in sections),
|
||||
"snippet": str(sections[0]["section_content"])[:360],
|
||||
"matched_heading": sections[0].get("heading"),
|
||||
"matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"type": "llmwiki_wiki",
|
||||
"mode": "citation",
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
|
||||
return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
|
||||
|
||||
async def _embed_query(self, query: str) -> np.ndarray:
|
||||
digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
cached = self._query_vectors.get(key)
|
||||
if cached is not None and monotonic() - cached[0] <= 300:
|
||||
self._query_vectors.move_to_end(key)
|
||||
return cached[1]
|
||||
vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
|
||||
key = f"{self.embedding.fingerprint}:{digest}"
|
||||
self._query_vectors[key] = (monotonic(), vector)
|
||||
self._query_vectors.move_to_end(key)
|
||||
while len(self._query_vectors) > 256:
|
||||
self._query_vectors.popitem(last=False)
|
||||
return vector
|
||||
|
||||
async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
|
||||
mapping_id = str(mapping["id"])
|
||||
try:
|
||||
snapshot = await self.cache.get(mapping_id, fingerprint)
|
||||
except ValueError as exc:
|
||||
await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
|
||||
if snapshot.matrix.size == 0:
|
||||
return []
|
||||
if snapshot.dimensions != vector.size:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
eligible = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, row in enumerate(snapshot.rows)
|
||||
if row["page"].get("status") != "draft" or (include_drafts and not external)
|
||||
],
|
||||
dtype=int,
|
||||
)
|
||||
if eligible.size == 0:
|
||||
return []
|
||||
indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
|
||||
terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
|
||||
candidates = []
|
||||
for index, score in zip(indices, scores, strict=True):
|
||||
row = snapshot.rows[int(index)]
|
||||
page = row["page"]
|
||||
vector_score = float(score)
|
||||
if vector_score < self.config.min_similarity:
|
||||
continue
|
||||
final_score = vector_score
|
||||
if self.config.keyword_boost_enabled and terms:
|
||||
haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
|
||||
keyword_score = sum(term in haystack for term in terms) / len(terms)
|
||||
weight = self.config.keyword_boost_weight
|
||||
final_score = (1 - weight) * vector_score + weight * keyword_score
|
||||
candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
|
||||
return candidates
|
||||
|
||||
def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
||||
scores = snapshot.matrix[eligible] @ vector
|
||||
count = min(self.config.top_k_sections, scores.size)
|
||||
if count <= 0:
|
||||
return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
|
||||
positions = np.argpartition(scores, -count)[-count:]
|
||||
positions = positions[np.argsort(scores[positions])[::-1]]
|
||||
return eligible[positions], scores[positions]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WikiSyncLeaseLost(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _safe_error(exc: BaseException) -> str:
|
||||
return f"{type(exc).__name__}: {str(exc)[:1000]}"
|
||||
|
||||
|
||||
class WikiSyncService:
|
||||
def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.embedding = embedding
|
||||
self.cache = cache
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
# Auto-dimension configurations need one successful response to
|
||||
# produce a stable fingerprint before lease/rebuild decisions.
|
||||
await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
|
||||
fingerprint = self.embedding.fingerprint
|
||||
revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
|
||||
if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
|
||||
await self.store.mark_rebuild_required([mapping_id], fingerprint)
|
||||
self.cache.invalidate(mapping_id)
|
||||
prior_state = "rebuild_required"
|
||||
rebuilding = prior_state == "rebuild_required"
|
||||
acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
|
||||
if not acquired:
|
||||
return {"mapping_id": mapping_id, "status": "already_running"}
|
||||
lease_lost = asyncio.Event()
|
||||
|
||||
async def _lease_heartbeat() -> None:
|
||||
interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
renewed = await self.store.renew_sync_lease(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.config.sync_lease_seconds,
|
||||
)
|
||||
except Exception:
|
||||
renewed = False
|
||||
logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
|
||||
if not renewed:
|
||||
lease_lost.set()
|
||||
return
|
||||
|
||||
heartbeat = asyncio.create_task(_lease_heartbeat())
|
||||
|
||||
def _ensure_lease() -> None:
|
||||
if lease_lost.is_set():
|
||||
raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
|
||||
|
||||
remote_count = 0
|
||||
changed = 0
|
||||
failed = 0
|
||||
try:
|
||||
remote_id = str(mapping["weknora_id"])
|
||||
await self.client.get_knowledge_base(remote_id)
|
||||
page_number = 1
|
||||
while True:
|
||||
batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
|
||||
pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
|
||||
remote_count += len(pages)
|
||||
for summary in pages:
|
||||
try:
|
||||
_ensure_lease()
|
||||
slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
|
||||
remote_page = await self.client.get_wiki_page(remote_id, slug)
|
||||
page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
|
||||
existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
|
||||
active_fingerprint = self.embedding.fingerprint
|
||||
if (
|
||||
not force
|
||||
and existing
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and not existing.get("is_remote_deleted")
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
continue
|
||||
sections = (
|
||||
[]
|
||||
if page["status"] == "archived"
|
||||
else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
|
||||
)
|
||||
embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
|
||||
fingerprint = self.embedding.fingerprint
|
||||
vectors = []
|
||||
for section, vector in zip(sections, embeddings, strict=True):
|
||||
blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
|
||||
vectors.append(
|
||||
{"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
|
||||
)
|
||||
_ensure_lease()
|
||||
await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
|
||||
self.cache.invalidate(mapping_id)
|
||||
changed += 1
|
||||
except Exception as exc:
|
||||
if isinstance(exc, WikiSyncLeaseLost):
|
||||
raise
|
||||
failed += 1
|
||||
try:
|
||||
fallback = normalize_wiki_page(mapping_id, summary)
|
||||
await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
|
||||
except Exception as record_exc:
|
||||
logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
|
||||
raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
|
||||
logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
_ensure_lease()
|
||||
total_pages = int(batch.get("total_pages") or 1)
|
||||
if page_number >= total_pages:
|
||||
break
|
||||
page_number += 1
|
||||
_ensure_lease()
|
||||
deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
|
||||
if deleted:
|
||||
self.cache.invalidate(mapping_id)
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=not (rebuilding and failed),
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
except asyncio.CancelledError:
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
owner=self.owner,
|
||||
sync_id=sync_id,
|
||||
success=False,
|
||||
remote_page_count=remote_count,
|
||||
error="Sync cancelled",
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
|
||||
logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
await asyncio.gather(heartbeat, return_exceptions=True)
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""Per-knowledge-base LRU snapshots for NumPy exact search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
|
||||
from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WikiVectorSnapshot:
|
||||
mapping_id: str
|
||||
index_revision: int
|
||||
embedding_fingerprint: str
|
||||
dimensions: int
|
||||
matrix: np.ndarray
|
||||
rows: list[dict[str, Any]]
|
||||
|
||||
|
||||
class WikiVectorCache:
|
||||
def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
|
||||
self.store = store
|
||||
self.max_knowledge_bases = max_knowledge_bases
|
||||
self.revision_check_seconds = revision_check_seconds
|
||||
self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
|
||||
self._checked_at: dict[str, float] = {}
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
def invalidate(self, mapping_id: str) -> None:
|
||||
self._snapshots.pop(mapping_id, None)
|
||||
self._checked_at.pop(mapping_id, None)
|
||||
|
||||
def clear(self) -> None:
|
||||
self._snapshots.clear()
|
||||
self._checked_at.clear()
|
||||
|
||||
async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
now = monotonic()
|
||||
if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
lock = self._locks.setdefault(mapping_id, asyncio.Lock())
|
||||
async with lock:
|
||||
revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
|
||||
self._checked_at[mapping_id] = monotonic()
|
||||
cached = self._snapshots.get(mapping_id)
|
||||
if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
|
||||
raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
|
||||
if active_fingerprint != fingerprint:
|
||||
raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
|
||||
if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
return cached
|
||||
raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
|
||||
snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
|
||||
self._snapshots[mapping_id] = snapshot
|
||||
self._snapshots.move_to_end(mapping_id)
|
||||
while len(self._snapshots) > self.max_knowledge_bases:
|
||||
evicted, _ = self._snapshots.popitem(last=False)
|
||||
self._checked_at.pop(evicted, None)
|
||||
return snapshot
|
||||
|
||||
@staticmethod
|
||||
def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
|
||||
rows = list(raw.get("rows") or [])
|
||||
if not rows:
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
|
||||
dimensions = int(rows[0]["embedding_dimensions"])
|
||||
vectors = []
|
||||
valid_rows = []
|
||||
for row in rows:
|
||||
if int(row["embedding_dimensions"]) != dimensions:
|
||||
raise ValueError("Mixed vector dimensions in one Wiki snapshot")
|
||||
vectors.append(decode_vector(row["vector_blob"], dimensions))
|
||||
valid_rows.append(row)
|
||||
matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
|
||||
matrix.setflags(write=False)
|
||||
return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
"""Validated little-endian normalized float32 Wiki vector codec."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
|
||||
vector = np.asarray(values, dtype=np.float32)
|
||||
if vector.ndim != 1 or vector.size == 0:
|
||||
raise ValueError("Embedding vector must be a non-empty one-dimensional array")
|
||||
if dimensions is not None and vector.size != dimensions:
|
||||
raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Embedding vector contains NaN or infinity")
|
||||
norm = float(np.linalg.norm(vector))
|
||||
if not np.isfinite(norm) or norm <= 0:
|
||||
raise ValueError("Embedding vector has zero norm")
|
||||
return np.asarray(vector / norm, dtype="<f4")
|
||||
|
||||
|
||||
def encode_vector(values, *, dimensions: int | None = None) -> tuple[bytes, int]:
|
||||
vector = normalize_vector(values, dimensions=dimensions)
|
||||
return vector.tobytes(order="C"), int(vector.size)
|
||||
|
||||
|
||||
def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
|
||||
if len(blob) != dimensions * 4:
|
||||
raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
|
||||
vector = np.frombuffer(blob, dtype="<f4")
|
||||
if not np.isfinite(vector).all():
|
||||
raise ValueError("Stored vector contains NaN or infinity")
|
||||
return vector.copy()
|
||||
+9
-1
@@ -22,7 +22,15 @@ class LlmWikiStore(ABC):
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
|
||||
+18
-6
@@ -33,6 +33,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"published_by": None,
|
||||
"reviewed_by": None,
|
||||
"published_at": None,
|
||||
"wiki_index_enabled": True,
|
||||
"external_search_enabled": False,
|
||||
"external_search_updated_at": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
@@ -65,7 +68,15 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
@@ -73,6 +84,11 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["name"] = name
|
||||
if description is not None:
|
||||
row["description"] = description
|
||||
if wiki_index_enabled is not None:
|
||||
row["wiki_index_enabled"] = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row["external_search_enabled"] = external_search_enabled
|
||||
row["external_search_updated_at"] = _now()
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
|
||||
@@ -143,11 +159,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if (
|
||||
row.get("owner_user_id") == "system"
|
||||
and row.get("name") == "对话沉淀"
|
||||
and row.get("publication_status") == "published"
|
||||
):
|
||||
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
|
||||
+4
-1
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -24,6 +24,9 @@ class LlmWikiKnowledgeBaseRow(Base):
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
|
||||
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
|
||||
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
+15
-2
@@ -19,7 +19,7 @@ def _now() -> datetime:
|
||||
|
||||
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("published_at", "created_at", "updated_at"):
|
||||
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
@@ -82,7 +82,15 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _to_dict(row) if row else None
|
||||
|
||||
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
if row is None:
|
||||
@@ -91,6 +99,11 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
row.name = name
|
||||
if description is not None:
|
||||
row.description = description
|
||||
if wiki_index_enabled is not None:
|
||||
row.wiki_index_enabled = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row.external_search_enabled = external_search_enabled
|
||||
row.external_search_updated_at = _now()
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""Persistence facade for the DeerFlow-local Wiki vector index."""
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory):
|
||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LlmWikiIndexStore",
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
]
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""In-memory local Wiki index store used by tests and DB-less mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
row = {
|
||||
**(existing or {}),
|
||||
**page,
|
||||
"id": page_id,
|
||||
"index_status": "ready",
|
||||
"indexed_fingerprint": fingerprint,
|
||||
"indexed_at": now,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"last_seen_sync_id": sync_id,
|
||||
"created_at": (existing or {}).get("created_at", now),
|
||||
"updated_at": now,
|
||||
}
|
||||
self.pages[page_id] = row
|
||||
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
|
||||
state = self._state(page["knowledge_base_mapping_id"])
|
||||
state["index_revision"] += 1
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": error,
|
||||
# Preserve the last atomically completed generation.
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
page.update({"is_remote_deleted": True, "index_status": "deleted"})
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
rows = []
|
||||
for page_id, vectors in self.vectors.items():
|
||||
page = self.pages[page_id]
|
||||
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
|
||||
continue
|
||||
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
|
||||
state = self._state(mapping_id)
|
||||
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
state = self._state(mapping_id)
|
||||
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
now = _now()
|
||||
expires = state.get("lease_expires_at")
|
||||
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
|
||||
return False
|
||||
state.update(
|
||||
{
|
||||
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
|
||||
"active_sync_id": sync_id,
|
||||
"lease_owner": owner,
|
||||
"lease_expires_at": now + timedelta(seconds=lease_seconds),
|
||||
"last_started_at": now,
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner:
|
||||
return False
|
||||
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
{
|
||||
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
|
||||
"lease_owner": None,
|
||||
"lease_expires_at": None,
|
||||
"active_sync_id": None,
|
||||
"last_error": error,
|
||||
"remote_page_count": remote_page_count,
|
||||
"local_page_count": len(pages),
|
||||
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
|
||||
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
}
|
||||
)
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
for mapping_id in mapping_ids:
|
||||
state = self._state(mapping_id)
|
||||
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
|
||||
page["index_status"] = "stale"
|
||||
return len(mapping_ids)
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class LlmWikiWikiPageRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_pages"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
|
||||
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
|
||||
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
|
||||
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
|
||||
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
|
||||
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
|
||||
|
||||
|
||||
class LlmWikiWikiVectorRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_vectors"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
|
||||
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
|
||||
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
|
||||
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+295
@@ -0,0 +1,295 @@
|
||||
"""SQLAlchemy implementation of the local Wiki index store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row) -> dict[str, Any]:
|
||||
value = row.to_dict()
|
||||
for key, item in tuple(value.items()):
|
||||
if isinstance(item, datetime):
|
||||
value[key] = item.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if row is None:
|
||||
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
else:
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
row.index_status = "ready"
|
||||
row.indexed_fingerprint = fingerprint
|
||||
row.indexed_at = now
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.updated_at = now
|
||||
await session.flush()
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||
session.add_all(
|
||||
[
|
||||
LlmWikiWikiVectorRow(
|
||||
id=str(uuid4()),
|
||||
page_id=row.id,
|
||||
knowledge_base_mapping_id=mapping_id,
|
||||
section_index=int(item["section_index"]),
|
||||
heading=item.get("heading"),
|
||||
section_content=str(item["section_content"]),
|
||||
content_hash=str(item["content_hash"]),
|
||||
embedding_fingerprint=fingerprint,
|
||||
embedding_dimensions=int(item["embedding_dimensions"]),
|
||||
vector_blob=bytes(item["vector_blob"]),
|
||||
created_at=now,
|
||||
)
|
||||
for item in vectors
|
||||
]
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
row.index_status = "failed"
|
||||
else:
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
|
||||
# Preserve the last atomically completed generation while the
|
||||
# newer remote version remains pending after an embedding error.
|
||||
row.index_status = "ready" if vector_count else "failed"
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = safe_error
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
now = _now()
|
||||
page_ids = [row.id for row in rows]
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
|
||||
for row in rows:
|
||||
row.is_remote_deleted = True
|
||||
row.index_status = "deleted"
|
||||
row.updated_at = now
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None:
|
||||
return 0, None, "not_synced"
|
||||
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
try:
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another worker created the one-per-KB state row between our
|
||||
# read and insert. Roll back this isolated bootstrap session;
|
||||
# the conditional UPDATE below is the actual lease election.
|
||||
await session.rollback()
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
or_(
|
||||
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at < now,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
|
||||
active_sync_id=sync_id,
|
||||
lease_owner=owner,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_started_at=now,
|
||||
last_error=None,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
|
||||
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
failed_count = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
*visible,
|
||||
or_(
|
||||
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
|
||||
LlmWikiWikiPageRow.last_error.is_not(None),
|
||||
),
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
)
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
|
||||
now = _now()
|
||||
was_rebuilding = state.state == "rebuilding"
|
||||
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
|
||||
state.active_sync_id = None
|
||||
state.lease_owner = None
|
||||
state.lease_expires_at = None
|
||||
state.last_error = (error or "")[:4000] or None
|
||||
state.remote_page_count = remote_page_count
|
||||
state.local_page_count = local_count
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
if mapping_ids:
|
||||
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
|
||||
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
if not mapping_ids:
|
||||
return 0
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
for mapping_id in mapping_ids:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.state = "rebuild_required"
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
|
||||
await session.commit()
|
||||
return len(mapping_ids)
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
|
||||
|
||||
Revision ID: 20260823_01
|
||||
Revises: 20260821_02
|
||||
Create Date: 2026-08-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = "20260823_01"
|
||||
down_revision = "20260821_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
if "wiki_index_enabled" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||
if "external_search_enabled" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||
if "external_search_updated_at" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
|
||||
mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
|
||||
if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
|
||||
op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
|
||||
if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
|
||||
op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
|
||||
|
||||
long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
|
||||
portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
|
||||
if not inspector.has_table("llmwiki_wiki_pages"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_pages",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("remote_page_id", sa.String(128), nullable=True),
|
||||
sa.Column("slug", sa.String(1024), nullable=False),
|
||||
sa.Column("slug_hash", sa.String(64), nullable=False),
|
||||
sa.Column("title", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("summary", long_text, nullable=False),
|
||||
sa.Column("content_md", long_text, nullable=False),
|
||||
sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="published"),
|
||||
sa.Column("aliases", portable_json, nullable=False),
|
||||
sa.Column("category_path", portable_json, nullable=False),
|
||||
sa.Column("parent_slug", sa.String(1024), nullable=True),
|
||||
sa.Column("wiki_path", sa.String(1024), nullable=True),
|
||||
sa.Column("source_refs", portable_json, nullable=False),
|
||||
sa.Column("chunk_refs", portable_json, nullable=False),
|
||||
sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False),
|
||||
sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
|
||||
sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
|
||||
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
|
||||
)
|
||||
op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
|
||||
op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
|
||||
|
||||
if not inspector.has_table("llmwiki_wiki_vectors"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_vectors",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
|
||||
sa.Column("section_index", sa.Integer(), nullable=False),
|
||||
sa.Column("heading", sa.String(1024), nullable=True),
|
||||
sa.Column("section_content", long_text, nullable=False),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False),
|
||||
sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
|
||||
sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
|
||||
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_sync_states",
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
|
||||
sa.Column("active_sync_id", sa.String(36), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(128), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
|
||||
if inspector.has_table(table):
|
||||
op.drop_table(table)
|
||||
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
|
||||
for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
|
||||
if index in indexes:
|
||||
op.drop_index(index, table_name="llmwiki_knowledge_bases")
|
||||
for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
|
||||
if column in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", column)
|
||||
+4
@@ -52,6 +52,7 @@ from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
@@ -129,6 +130,9 @@ __all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"MenuOverrideRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
|
||||
+23
-51
@@ -6,28 +6,25 @@ import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langgraph.typing import ContextT
|
||||
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
|
||||
from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
|
||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||
from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
|
||||
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
|
||||
|
||||
|
||||
_CONVERSATION_DEPOSIT_NAME = "对话沉淀"
|
||||
_CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
|
||||
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
|
||||
return (
|
||||
str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
|
||||
and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
|
||||
)
|
||||
return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
|
||||
|
||||
|
||||
def _sanitize_conversation_deposit_value(value: Any) -> Any:
|
||||
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
|
||||
query: Focused natural-language retrieval query.
|
||||
"""
|
||||
|
||||
local_config = get_app_config().llmwiki.local_wiki_index
|
||||
resolved = get_resolved_llmwiki_runtime()
|
||||
if not resolved.weknora_enabled:
|
||||
return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
|
||||
if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
|
||||
return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
|
||||
|
||||
store = get_llmwiki_store()
|
||||
user_id = get_effective_user_id()
|
||||
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
|
||||
return "No authorized LLMWiki knowledge bases are available."
|
||||
|
||||
try:
|
||||
results = await build_weknora_client(resolved).search(
|
||||
query.strip(),
|
||||
[str(row["weknora_id"]) for row in rows],
|
||||
)
|
||||
except (RuntimeError, WeKnoraError) as exc:
|
||||
return f"LLMWiki retrieval failed: {exc}"
|
||||
|
||||
local_by_remote = {str(row["weknora_id"]): row for row in rows}
|
||||
citations = []
|
||||
for result in results:
|
||||
mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
|
||||
if mapping is None:
|
||||
continue
|
||||
visible_result = (
|
||||
_sanitize_conversation_deposit_value(result)
|
||||
if _is_conversation_deposit_mapping(mapping)
|
||||
else result
|
||||
)
|
||||
params = urlencode(
|
||||
{
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"chunk_id": visible_result.get("chunk_id") or "",
|
||||
"knowledge_id": visible_result.get("knowledge_id") or "",
|
||||
"document_id": visible_result.get("knowledge_id") or "",
|
||||
"wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
|
||||
}
|
||||
)
|
||||
citations.append(
|
||||
{
|
||||
"title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
|
||||
"content": visible_result.get("content") or "",
|
||||
"score": visible_result.get("score") or 0,
|
||||
"knowledge_base_id": mapping["id"],
|
||||
"knowledge_base_name": mapping.get("name") or "",
|
||||
"source": mapping.get("name") or "LLMWiki",
|
||||
"knowledge_id": visible_result.get("knowledge_id") or "",
|
||||
"chunk_id": visible_result.get("chunk_id") or "",
|
||||
"url": f"/weknora-source-preview?{params}",
|
||||
}
|
||||
)
|
||||
payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
|
||||
except (RuntimeError, WikiIndexError) as exc:
|
||||
code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
|
||||
return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
|
||||
mapping_by_id = {str(row["id"]): row for row in rows}
|
||||
results = []
|
||||
for item in payload["results"]:
|
||||
mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
|
||||
visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
|
||||
results.append(visible)
|
||||
return json.dumps(
|
||||
{
|
||||
"query": query,
|
||||
"selection": [row["id"] for row in rows],
|
||||
"knowledge_base_count": len(rows),
|
||||
"results": citations,
|
||||
"mode": payload.get("mode") or "wiki_api",
|
||||
"results": results,
|
||||
"partial": payload.get("partial", False),
|
||||
"warnings": payload.get("warnings", []),
|
||||
"fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -21,7 +21,8 @@ dependencies = [
|
||||
"langgraph-cli>=0.4.24",
|
||||
"langgraph-runtime-inmem>=0.28.0",
|
||||
"markdownify>=1.2.2",
|
||||
"markitdown[all,xlsx]>=0.0.1a2",
|
||||
"markitdown[all,xlsx]>=0.0.1a2",
|
||||
"numpy>=2.0.0",
|
||||
"pydantic>=2.12.5",
|
||||
"pyyaml>=6.0.3",
|
||||
"readabilipy>=0.3.0",
|
||||
|
||||
Reference in new issue
Block a user