feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
+15
@@ -501,6 +501,21 @@ class WeKnoraClient:
|
||||
raise WeKnoraError("WeKnora returned an invalid wiki graph")
|
||||
return dict(value)
|
||||
|
||||
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
|
||||
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
|
||||
if not isinstance(value, dict):
|
||||
raise WeKnoraError("Invalid Wiki generation status")
|
||||
return value
|
||||
|
||||
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
|
||||
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
|
||||
if not isinstance(value, dict) or not value.get("id"):
|
||||
raise WeKnoraError("Invalid Wiki folder response")
|
||||
return value
|
||||
|
||||
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
|
||||
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
|
||||
|
||||
async def list_wiki_pages(
|
||||
self,
|
||||
knowledge_base_id: str,
|
||||
|
||||
+8
@@ -105,4 +105,12 @@ def chunk_wiki_page(
|
||||
context.append(f"章节:{heading}")
|
||||
embedding_text = "\n".join([*context, "", chunk]).strip()
|
||||
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
|
||||
# A short article with a short summary must not disappear merely because
|
||||
# both sections fall below min_chars. Keep one combined, useful vector.
|
||||
if not result and (content or summary):
|
||||
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
|
||||
normalized = re.sub(r"\s+", " ", fallback).strip()
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
prefix = prefix_lines if title_prefix_enabled else []
|
||||
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
|
||||
return result
|
||||
+47
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
|
||||
import httpx
|
||||
@@ -43,6 +44,8 @@ class StrictWikiEmbeddingClient:
|
||||
self._client = http_client
|
||||
self._owns_client = http_client is None
|
||||
self._resolved_dimensions = config.dimensions
|
||||
self._local_model = None
|
||||
self._local_lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def dimensions(self) -> int | None:
|
||||
@@ -64,6 +67,11 @@ class StrictWikiEmbeddingClient:
|
||||
return self._client
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
if self.config.provider == "local":
|
||||
if not texts:
|
||||
return []
|
||||
async with self._local_lock:
|
||||
return await asyncio.to_thread(self._embed_local, texts)
|
||||
if not self.config.enabled or not self.config.base_url or not self.config.model:
|
||||
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
|
||||
if not texts:
|
||||
@@ -76,6 +84,45 @@ class StrictWikiEmbeddingClient:
|
||||
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
|
||||
return vectors
|
||||
|
||||
def _embed_local(self, texts: list[str]) -> list[list[float]]:
|
||||
if not self.config.enabled:
|
||||
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
|
||||
if self._local_model is None:
|
||||
try:
|
||||
from fastembed import TextEmbedding
|
||||
except ImportError as exc:
|
||||
raise WikiEmbeddingError("请安装本地编码依赖:uv sync --extra local-embedding", retryable=False) from exc
|
||||
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
|
||||
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
|
||||
from fastembed.common.model_description import ModelSource, PoolingType
|
||||
|
||||
TextEmbedding.add_custom_model(
|
||||
model=name,
|
||||
pooling=PoolingType.CLS,
|
||||
normalization=True,
|
||||
dim=1024,
|
||||
sources=ModelSource(hf="BAAI/bge-m3"),
|
||||
model_file="onnx/model.onnx",
|
||||
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
|
||||
license="mit",
|
||||
size_in_gb=2.29,
|
||||
)
|
||||
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
|
||||
options = {"model_name": name, "threads": self.config.local_threads, "cache_dir": str(cache_dir), "providers": ["CPUExecutionProvider"]}
|
||||
if self.config.local_model_path:
|
||||
options["specific_model_path"] = str(Path(self.config.local_model_path).expanduser())
|
||||
options["local_files_only"] = True
|
||||
self._local_model = TextEmbedding(**options)
|
||||
values = []
|
||||
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
|
||||
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
|
||||
if self._resolved_dimensions is None:
|
||||
self._resolved_dimensions = int(normalized.size)
|
||||
values.append(normalized.tolist())
|
||||
if len(values) != len(texts):
|
||||
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
|
||||
return values
|
||||
|
||||
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
url = self.config.base_url.rstrip("/") + "/embeddings"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
+7
-2
@@ -36,8 +36,9 @@ class WikiSyncService:
|
||||
self.cache = cache
|
||||
self.client = client
|
||||
self.owner = f"{platform.node()}:{uuid4()}"
|
||||
self.on_completed = None
|
||||
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
|
||||
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
|
||||
mapping_id = str(mapping["id"])
|
||||
sync_id = str(uuid4())
|
||||
if self.embedding.dimensions is None:
|
||||
@@ -111,6 +112,7 @@ class WikiSyncService:
|
||||
and existing.get("content_hash") == page["content_hash"]
|
||||
and existing.get("indexed_fingerprint") == active_fingerprint
|
||||
and existing.get("index_status") == "ready"
|
||||
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
|
||||
and not existing.get("is_remote_deleted")
|
||||
):
|
||||
await self.store.touch_page(str(existing["id"]), sync_id)
|
||||
@@ -160,7 +162,10 @@ class WikiSyncService:
|
||||
remote_page_count=remote_count,
|
||||
error=f"{failed} page(s) failed" if failed else None,
|
||||
)
|
||||
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
|
||||
if not failed and remote_count and notify and self.on_completed:
|
||||
result["assistant_sync"] = await self.on_completed(mapping)
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
await self.store.finish_sync(
|
||||
mapping_id,
|
||||
|
||||
Reference in new issue
Block a user