feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -0,0 +1,116 @@
from __future__ import annotations
import pytest
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
class FakeEmbedding:
dimensions = 2
fingerprint = "fp"
def __init__(self) -> None:
self.calls = 0
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
self.calls += 1
return [[1.0, 0.0] for _ in texts]
class FailingEmbedding(FakeEmbedding):
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
raise RuntimeError("embedding unavailable")
class NewFingerprintEmbedding(FakeEmbedding):
fingerprint = "fp-new"
class FakeWeKnora:
def __init__(self) -> None:
self.pages = {
"one": {
"id": "p1",
"slug": "one",
"title": "One",
"summary": "A long enough summary for indexing and retrieval.",
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
"status": "published",
"version": 1,
},
}
async def get_knowledge_base(self, knowledge_base_id: str):
return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
return dict(self.pages[slug])
@pytest.mark.asyncio
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
embedding = FakeEmbedding()
client = FakeWeKnora()
service = WikiSyncService(config, store, embedding, WikiVectorCache(store), client)
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
first = await service.sync_mapping(mapping)
assert first["changed_pages"] == 1
assert embedding.calls == 1
second = await service.sync_mapping(mapping)
assert second["changed_pages"] == 0
assert embedding.calls == 1
client.pages.clear()
deleted = await service.sync_mapping(mapping)
assert deleted["deleted_pages"] == 1
assert next(iter(store.pages.values()))["is_remote_deleted"] is True
@pytest.mark.asyncio
async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
client = FakeWeKnora()
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
page = next(iter(store.pages.values()))
old_hash = page["content_hash"]
old_vectors = list(store.vectors[page["id"]])
client.pages["one"]["content"] += " Changed."
result = await WikiSyncService(config, store, FailingEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
assert result["status"] == "partial"
assert store.states["kb"]["state"] == "partial"
assert store.states["kb"]["failed_page_count"] == 1
assert store.pages[page["id"]]["content_hash"] == old_hash
assert store.pages[page["id"]]["index_status"] == "ready"
assert store.vectors[page["id"]] == old_vectors
@pytest.mark.asyncio
async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
client = FakeWeKnora()
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
embedding = NewFingerprintEmbedding()
result = await WikiSyncService(config, store, embedding, WikiVectorCache(store), client).sync_mapping(mapping)
assert result["status"] == "completed"
assert result["changed_pages"] == 1
assert embedding.calls == 1
assert store.states["kb"]["state"] == "idle"
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}