feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -501,6 +501,21 @@ class WeKnoraClient:
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki generation status")
return value
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("Invalid Wiki folder response")
return value
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
async def list_wiki_pages(
self,
knowledge_base_id: str,
@@ -105,4 +105,12 @@ def chunk_wiki_page(
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
# A short article with a short summary must not disappear merely because
# both sections fall below min_chars. Keep one combined, useful vector.
if not result and (content or summary):
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
normalized = re.sub(r"\s+", " ", fallback).strip()
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
prefix = prefix_lines if title_prefix_enabled else []
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
return result
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import hashlib
import logging
from pathlib import Path
from time import monotonic
import httpx
@@ -43,6 +44,8 @@ class StrictWikiEmbeddingClient:
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_lock = asyncio.Lock()
@property
def dimensions(self) -> int | None:
@@ -64,6 +67,11 @@ class StrictWikiEmbeddingClient:
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
async with self._local_lock:
return await asyncio.to_thread(self._embed_local, texts)
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
@@ -76,6 +84,45 @@ class StrictWikiEmbeddingClient:
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
if self.config.local_model_path:
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
options["local_files_only"] = True
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
if len(values) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
@@ -36,8 +36,9 @@ class WikiSyncService:
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
@@ -111,6 +112,7 @@ class WikiSyncService:
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
@@ -160,7 +162,10 @@ class WikiSyncService:
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
if not failed and remote_count and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,