feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
|
||||
|
||||
class FakeEmbedding:
|
||||
dimensions = 2
|
||||
fingerprint = "fp"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
self.calls += 1
|
||||
return [[1.0, 0.0] for _ in texts]
|
||||
|
||||
|
||||
class FailingEmbedding(FakeEmbedding):
|
||||
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
|
||||
raise RuntimeError("embedding unavailable")
|
||||
|
||||
|
||||
class NewFingerprintEmbedding(FakeEmbedding):
|
||||
fingerprint = "fp-new"
|
||||
|
||||
|
||||
class FakeWeKnora:
|
||||
def __init__(self) -> None:
|
||||
self.pages = {
|
||||
"one": {
|
||||
"id": "p1",
|
||||
"slug": "one",
|
||||
"title": "One",
|
||||
"summary": "A long enough summary for indexing and retrieval.",
|
||||
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
|
||||
"status": "published",
|
||||
"version": 1,
|
||||
},
|
||||
}
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str):
|
||||
return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
|
||||
|
||||
async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
|
||||
return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
|
||||
|
||||
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
|
||||
return dict(self.pages[slug])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
embedding = FakeEmbedding()
|
||||
client = FakeWeKnora()
|
||||
service = WikiSyncService(config, store, embedding, WikiVectorCache(store), client)
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
|
||||
first = await service.sync_mapping(mapping)
|
||||
assert first["changed_pages"] == 1
|
||||
assert embedding.calls == 1
|
||||
second = await service.sync_mapping(mapping)
|
||||
assert second["changed_pages"] == 0
|
||||
assert embedding.calls == 1
|
||||
|
||||
client.pages.clear()
|
||||
deleted = await service.sync_mapping(mapping)
|
||||
assert deleted["deleted_pages"] == 1
|
||||
assert next(iter(store.pages.values()))["is_remote_deleted"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
client = FakeWeKnora()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
|
||||
page = next(iter(store.pages.values()))
|
||||
old_hash = page["content_hash"]
|
||||
old_vectors = list(store.vectors[page["id"]])
|
||||
client.pages["one"]["content"] += " Changed."
|
||||
|
||||
result = await WikiSyncService(config, store, FailingEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
|
||||
|
||||
assert result["status"] == "partial"
|
||||
assert store.states["kb"]["state"] == "partial"
|
||||
assert store.states["kb"]["failed_page_count"] == 1
|
||||
assert store.pages[page["id"]]["content_hash"] == old_hash
|
||||
assert store.pages[page["id"]]["index_status"] == "ready"
|
||||
assert store.vectors[page["id"]] == old_vectors
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> None:
|
||||
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
client = FakeWeKnora()
|
||||
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
|
||||
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
|
||||
|
||||
embedding = NewFingerprintEmbedding()
|
||||
result = await WikiSyncService(config, store, embedding, WikiVectorCache(store), client).sync_mapping(mapping)
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert result["changed_pages"] == 1
|
||||
assert embedding.calls == 1
|
||||
assert store.states["kb"]["state"] == "idle"
|
||||
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
|
||||
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
|
||||
Reference in new issue
Block a user