feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -0,0 +1,51 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from starlette.requests import Request
from app.gateway.auth_middleware import _authorize_external_llmwiki
from app.gateway.routers.external_llmwiki import _external_mappings
def _request(key: str) -> Request:
return Request({"type": "http", "method": "POST", "path": "/api/external/llmwiki/wiki/vector-search", "headers": [(b"x-api-key", key.encode())]})
def test_external_api_key_uses_isolated_fail_closed_auth(monkeypatch) -> None:
config = SimpleNamespace(enabled=True, api_key="current", previous_api_key="previous")
monkeypatch.setattr("deerflow.config.get_app_config", lambda: SimpleNamespace(llmwiki=SimpleNamespace(local_wiki_index=SimpleNamespace(external_api=config))))
assert _authorize_external_llmwiki(_request("current")) is None
assert _authorize_external_llmwiki(_request("previous")) is None
rejected = _authorize_external_llmwiki(_request("wrong"))
assert rejected is not None
assert rejected.status_code == 401
@pytest.mark.asyncio
async def test_external_mappings_require_explicit_public_external_permission() -> None:
rows = [
{"id": "allowed", "publication_status": "published", "external_search_enabled": True, "wiki_index_enabled": True},
{"id": "private", "publication_status": "private", "external_search_enabled": True, "wiki_index_enabled": True},
{"id": "not-opted-in", "publication_status": "published", "external_search_enabled": False, "wiki_index_enabled": True},
{
"id": "deposit",
"name": "对话沉淀",
"owner_user_id": "system",
"publication_status": "published",
"external_search_enabled": True,
"wiki_index_enabled": True,
},
]
class Store:
async def list_visible(self, *_args, **_kwargs):
return rows
request = Request({"type": "http", "method": "POST", "path": "/", "headers": [], "app": SimpleNamespace(state=SimpleNamespace(llmwiki_store=Store()))})
assert [row["id"] for row in await _external_mappings(request, [])] == ["allowed"]
with pytest.raises(HTTPException) as exc_info:
await _external_mappings(request, ["not-opted-in"])
assert exc_info.value.status_code == 404
@@ -0,0 +1,68 @@
from __future__ import annotations
from datetime import UTC, datetime
import pytest
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector
from deerflow.persistence.base import Base
from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
@pytest.mark.asyncio
async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(tmp_path) -> None:
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'wiki.db'}")
async with engine.begin() as connection:
await connection.run_sync(
lambda sync_connection: Base.metadata.create_all(
sync_connection,
tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiSyncStateRow.__table__],
)
)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
async with session_factory() as session:
session.add(
LlmWikiKnowledgeBaseRow(
id="mapping-1",
weknora_id="remote-1",
owner_user_id="u1",
name="Wiki",
description="",
kb_type="document",
publication_status="private",
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
)
await session.commit()
store = SqlLlmWikiIndexStore(session_factory)
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
saved = await store.replace_page_vectors(
page=page,
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
fingerprint="fp",
sync_id="sync-1",
)
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
assert saved["slug"] == "people/alice"
assert len(snapshot["rows"]) == 1
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
await engine.dispose()
def test_vector_codec_rejects_corrupt_and_non_finite_vectors() -> None:
with pytest.raises(ValueError, match="NaN"):
encode_vector([1.0, float("nan")], dimensions=2)
with pytest.raises(ValueError, match="Corrupt"):
decode_vector(b"123", 2)
@@ -0,0 +1,89 @@
from __future__ import annotations
import pytest
from pydantic import ValidationError
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
class FakeEmbedding:
dimensions = 2
fingerprint = "fp"
async def embed_query(self, text: str) -> list[float]:
return [1.0, 0.0]
def test_enabled_index_requires_supported_enabled_embedding() -> None:
with pytest.raises(ValidationError, match="embedding must be enabled"):
LocalWikiIndexConfig(enabled=True, embedding={"enabled": False})
with pytest.raises(ValidationError, match="provider must be openai"):
LocalWikiIndexConfig(embedding={"provider": "custom"})
@pytest.mark.asyncio
async def test_exact_search_groups_sections_by_wiki_page_and_filters_drafts() -> None:
store = MemoryLlmWikiIndexStore()
await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
published = normalize_wiki_page("kb", {"slug": "alice", "title": "Alice", "summary": "Founder", "content": "Alpha", "status": "published"})
draft = normalize_wiki_page("kb", {"slug": "secret", "title": "Secret", "content": "Draft", "status": "draft"})
a, _ = encode_vector([1.0, 0.0], dimensions=2)
b, _ = encode_vector([0.9, 0.1], dimensions=2)
await store.replace_page_vectors(
page=published,
vectors=[
{"section_index": 0, "heading": "A", "section_content": "Alpha", "content_hash": "a", "embedding_dimensions": 2, "vector_blob": a},
{"section_index": 1, "heading": "B", "section_content": "Beta", "content_hash": "b", "embedding_dimensions": 2, "vector_blob": b},
],
fingerprint="fp",
sync_id="s",
)
await store.replace_page_vectors(page=draft, vectors=[{"section_index": 0, "heading": None, "section_content": "Draft", "content_hash": "d", "embedding_dimensions": 2, "vector_blob": a}], fingerprint="fp", sync_id="s")
config = LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8, max_sections_per_page=2)
service = WikiVectorSearchService(config, FakeEmbedding(), WikiVectorCache(store, revision_check_seconds=0))
mapping = {"id": "kb", "name": "People"}
result = await service.search("alice", [mapping])
assert [item["wiki_slug"] for item in result["results"]] == ["alice"]
assert len(result["results"][0]["matched_sections"]) == 2
assert "wiki_slug=alice" in result["results"][0]["url"]
with_drafts = await service.search("alice", [mapping], include_drafts=True)
assert {item["wiki_slug"] for item in with_drafts["results"]} == {"alice", "secret"}
narrow_service = WikiVectorSearchService(
LocalWikiIndexConfig(min_similarity=0, top_k_sections=1, top_k_pages=1),
FakeEmbedding(),
WikiVectorCache(store, revision_check_seconds=0),
)
assert [item["wiki_slug"] for item in (await narrow_service.search("alice", [mapping]))["results"]] == ["alice"]
@pytest.mark.asyncio
async def test_corrupt_blob_marks_index_for_rebuild() -> None:
store = MemoryLlmWikiIndexStore()
await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
page = normalize_wiki_page("kb", {"slug": "broken", "title": "Broken", "content": "Broken", "status": "published"})
blob, _ = encode_vector([1.0, 0.0], dimensions=2)
saved = await store.replace_page_vectors(
page=page,
vectors=[{"section_index": 0, "heading": None, "section_content": "Broken", "content_hash": "x", "embedding_dimensions": 2, "vector_blob": blob}],
fingerprint="fp",
sync_id="s",
)
store.vectors[saved["id"]][0]["vector_blob"] = b"broken"
service = WikiVectorSearchService(
LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8),
FakeEmbedding(),
WikiVectorCache(store, revision_check_seconds=0),
)
with pytest.raises(WikiIndexError) as exc_info:
await service.search("broken", [{"id": "kb", "name": "Broken"}])
assert exc_info.value.code == "WIKI_INDEX_REBUILDING"
assert store.states["kb"]["state"] == "rebuild_required"
@@ -0,0 +1,154 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from openpyxl import load_workbook
from app.gateway.routers.llmwiki import _build_wiki_excel
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
class _Store:
def __init__(self, states):
self.states = states
async def list_index_status(self, mapping_ids=None):
wanted = set(mapping_ids or [])
return [state for state in self.states if not wanted or state["knowledge_base_mapping_id"] in wanted]
class _WikiClient:
def __init__(self):
self.list_calls = []
self.raw_search_called = False
async def list_wiki_pages(self, knowledge_base_id, *, page=1, page_size=50, query=""):
self.list_calls.append((knowledge_base_id, query))
return {"pages": [{"slug": "guides/install", "title": "安装"}], "total": 1, "total_pages": 1}
async def get_wiki_page(self, knowledge_base_id, slug):
return {
"id": f"{knowledge_base_id}:{slug}",
"slug": slug,
"title": "安装",
"summary": "Wiki 摘要",
"content": "# 安装\n这是处理后的 Wiki 正文。",
"status": "published",
"category_path": ["指南"],
}
async def search(self, query, knowledge_base_ids):
self.raw_search_called = True
raise AssertionError("raw knowledge search must never be called")
class _VectorSearch:
def __init__(self):
self.embedding = SimpleNamespace(fingerprint="fp-current")
self.calls = []
async def search(self, query, mappings, **kwargs):
self.calls.append([mapping["id"] for mapping in mappings])
mapping = mappings[0]
return {
"results": [
{
"kind": "wiki_page",
"id": "local-page",
"wiki_page_id": "local-page",
"wiki_slug": "local/page",
"knowledge_base_id": mapping["id"],
"title": "本地页面",
"content": "本地 Wiki",
"matched_sections": [],
"score": 0.95,
}
],
"warnings": [],
}
def _ready_state(mapping_id: str):
return {
"knowledge_base_mapping_id": mapping_id,
"state": "idle",
"last_success_at": "2026-08-24T00:00:00Z",
"embedding_fingerprint": "fp-current",
"local_page_count": 1,
"ready_page_count": 1,
"failed_page_count": 0,
"vector_count": 3,
}
def test_fully_vectorized_requires_completed_full_consistent_scan() -> None:
state = _ready_state("kb")
assert is_fully_vectorized(state, "fp-current")
assert not is_fully_vectorized({**state, "state": "partial"}, "fp-current")
assert not is_fully_vectorized({**state, "ready_page_count": 0}, "fp-current")
assert not is_fully_vectorized(state, "different-fingerprint")
@pytest.mark.asyncio
async def test_unvectorized_library_uses_only_wiki_page_search_api() -> None:
client = _WikiClient()
service = WikiRetrievalService(_Store([]), vector_search=_VectorSearch(), client=client)
mapping = {"id": "local-kb", "weknora_id": "remote-kb", "name": "测试库"}
result = await service.search("安装", [mapping])
assert result["mode"] == "wiki_api"
assert result["fallback_knowledge_base_ids"] == ["local-kb"]
assert result["results"][0]["kind"] == "wiki_page"
assert result["results"][0]["content"] == "# 安装\n这是处理后的 Wiki 正文。"
assert "chunk_id" not in result["results"][0]
assert client.list_calls == [("remote-kb", "安装")]
assert not client.raw_search_called
@pytest.mark.asyncio
async def test_mixed_search_vectors_complete_library_and_falls_back_incomplete_library() -> None:
client = _WikiClient()
vector = _VectorSearch()
service = WikiRetrievalService(_Store([_ready_state("ready")]), vector_search=vector, client=client)
mappings = [
{"id": "ready", "weknora_id": "remote-ready", "name": "已完成"},
{"id": "pending", "weknora_id": "remote-pending", "name": "未完成"},
]
result = await service.search("安装", mappings, top_k_pages=8)
assert result["mode"] == "mixed"
assert vector.calls == [["ready"]]
assert client.list_calls == [("remote-pending", "安装")]
assert {item["knowledge_base_id"] for item in result["results"]} == {"ready", "pending"}
def test_excel_export_contains_current_directory_and_every_parent_directory() -> None:
long_content = "正文" * 20_000
pages = [
{"id": "1", "slug": "root", "title": "根专题", "content": "根", "status": "published"},
{"id": "2", "slug": "root/child", "title": "子专题", "parent_slug": "root", "content": "子", "status": "published"},
{
"id": "3",
"slug": "root/child/article",
"title": "文章",
"parent_slug": "root/child",
"content": long_content,
"status": "published",
},
]
workbook = load_workbook(_build_wiki_excel("知识库", pages), read_only=True)
sheet = workbook["Wiki 全量导出"]
rows = list(sheet.iter_rows(values_only=True))
headings = list(rows[0])
article = dict(zip(headings, rows[3], strict=True))
assert article["目录完整路径"] == "根专题 / 子专题"
assert article["所在目录"] == "子专题"
assert article["父级目录完整路径"] == "根专题"
assert article["第1级目录"] == "根专题"
assert article["第2级目录"] == "子专题"
assert article["正文"] + article["正文(续1)"] == long_content
@@ -0,0 +1,116 @@
from __future__ import annotations
import pytest
from deerflow.config.llmwiki_config import LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
class FakeEmbedding:
dimensions = 2
fingerprint = "fp"
def __init__(self) -> None:
self.calls = 0
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
self.calls += 1
return [[1.0, 0.0] for _ in texts]
class FailingEmbedding(FakeEmbedding):
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
raise RuntimeError("embedding unavailable")
class NewFingerprintEmbedding(FakeEmbedding):
fingerprint = "fp-new"
class FakeWeKnora:
def __init__(self) -> None:
self.pages = {
"one": {
"id": "p1",
"slug": "one",
"title": "One",
"summary": "A long enough summary for indexing and retrieval.",
"content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
"status": "published",
"version": 1,
},
}
async def get_knowledge_base(self, knowledge_base_id: str):
return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
async def get_wiki_page(self, knowledge_base_id: str, slug: str):
return dict(self.pages[slug])
@pytest.mark.asyncio
async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
embedding = FakeEmbedding()
client = FakeWeKnora()
service = WikiSyncService(config, store, embedding, WikiVectorCache(store), client)
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
first = await service.sync_mapping(mapping)
assert first["changed_pages"] == 1
assert embedding.calls == 1
second = await service.sync_mapping(mapping)
assert second["changed_pages"] == 0
assert embedding.calls == 1
client.pages.clear()
deleted = await service.sync_mapping(mapping)
assert deleted["deleted_pages"] == 1
assert next(iter(store.pages.values()))["is_remote_deleted"] is True
@pytest.mark.asyncio
async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
client = FakeWeKnora()
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
page = next(iter(store.pages.values()))
old_hash = page["content_hash"]
old_vectors = list(store.vectors[page["id"]])
client.pages["one"]["content"] += " Changed."
result = await WikiSyncService(config, store, FailingEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
assert result["status"] == "partial"
assert store.states["kb"]["state"] == "partial"
assert store.states["kb"]["failed_page_count"] == 1
assert store.pages[page["id"]]["content_hash"] == old_hash
assert store.pages[page["id"]]["index_status"] == "ready"
assert store.vectors[page["id"]] == old_vectors
@pytest.mark.asyncio
async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> None:
config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
store = MemoryLlmWikiIndexStore()
client = FakeWeKnora()
mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
embedding = NewFingerprintEmbedding()
result = await WikiSyncService(config, store, embedding, WikiVectorCache(store), client).sync_mapping(mapping)
assert result["status"] == "completed"
assert result["changed_pages"] == 1
assert embedding.calls == 1
assert store.states["kb"]["state"] == "idle"
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}