feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -0,0 +1,207 @@
import base64
import sys
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers import wiki_packages
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import write_package
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.base import Base
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
@pytest.mark.asyncio
async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp_path, monkeypatch):
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
blob, _ = encode_vector([0.3, 0.5, 0.7])
pages = [
{
"slug": "concept/cooling",
"title": "散热",
"page_type": "concept",
"content": "散热管理用于降低设备温度。参见 [[entity/fan]]",
"summary": "设备散热",
"tags": ["设备", "维护"],
"category_path": ["运维", "硬件"],
"aliases": ["降温"],
"out_links": ["entity/fan"],
"in_links": [],
},
{"slug": "entity/fan", "title": "风扇", "page_type": "entity", "content": "风扇通过气流帮助散热,参见 [[concept/cooling]]。", "aliases": [], "out_links": ["concept/cooling"], "in_links": ["concept/cooling"]},
]
vectors = [
{
"wiki_slug": p["slug"],
"section_index": 0,
"section_content": p["content"],
"content_hash": "test",
"embedding_model": "test-model",
"embedding_fingerprint": "test-fp",
"embedding_dimensions": 3,
"vector_blob_base64": base64.b64encode(blob).decode(),
}
for p in pages
]
original = tmp_path / "original.jsonl"
entities = [{"id": p["slug"], "name": p["title"], "type": p["page_type"], "source_wiki_slug": p["slug"]} for p in pages]
relations = [{"source_entity_id": pages[0]["slug"], "target_entity_id": pages[1]["slug"], "source_wiki_slug": pages[0]["slug"], "predicate": "references"}]
write_package(original, [
("source", {"provider": "weknora", "package_version": 1}),
*[("wiki_page", p) for p in pages],
*[("vector", v) for v in vectors],
*[("entity", e) for e in entities],
*[("relation", r) for r in relations],
("chunk", {"id": "raw-chunk", "content": "原始文档分段不是 Wiki 页面"}),
])
archive = PackageArchive(original)
base = await store.initialize(created_by="admin")
job = await store.import_package(package=archive, source_type="weknora", source_key="source", source_name="来源", trigger="test", created_by="admin")
assert job["counts"]["vectors"] == 2
assert job["counts"]["wiki_pages"] == 2
assert job["counts"]["chunk_pages"] == 1
imported = await store.get_page(pages[0]["slug"])
for field in ("category_path", "tags", "aliases", "out_links"):
assert imported[field] == pages[0][field]
assert imported["content_markdown"] == pages[0]["content"]
# The global base may receive the same Wiki through multiple libraries.
# Only the current article's import supplies vectors; older contributions
# remain auditable but must not duplicate/stale the restored index.
await store.import_package(package=archive, source_type="weknora", source_key="second-source", source_name="第二来源", trigger="test", created_by="admin")
exported = tmp_path / "export.jsonl"
write_package(exported, [record async for record in store.iter_export_records(base["id"])])
restored_archive = PackageArchive(exported)
assert {v["vector_blob_base64"] for v in restored_archive["vectors"]} == {base64.b64encode(blob).decode()}
assert sorted((v["wiki_slug"], v["section_index"]) for v in restored_archive["vectors"]) == sorted((v["wiki_slug"], v["section_index"]) for v in vectors)
assert all(r["source_wiki_slug"] in {p["slug"] for p in pages} for r in restored_archive["relations"])
index = MemoryLlmWikiIndexStore()
received = []
class Remote:
async def create_wiki_folder(self, kb, *, name, parent_id=""):
return {"id": f"{parent_id}/{name}"}
async def create_wiki_page(self, kb, page):
received.append(page)
return page
async def rebuild_wiki_links(self, kb):
pass
class Embedding:
dimensions = 3
fingerprint = "test-fp"
async def embed_query(self, query):
raise AssertionError("Existing package must not be encoded")
app = FastAPI()
app.state.llmwiki_index_store = index
app.state.llmwiki_embedding = Embedding()
app.state.llmwiki_vector_cache = None
app.state.config = None
monkeypatch.setattr(wiki_packages, "get_resolved_llmwiki_runtime", lambda c: None)
monkeypatch.setattr(wiki_packages, "build_weknora_client", lambda c: Remote())
monkeypatch.setattr(wiki_packages, "_package_root", lambda: tmp_path)
from app.gateway import knowledge_transfer
async def sync(app, mapping):
return {"status": "completed"}
monkeypatch.setattr(knowledge_transfer, "sync_to_global", sync)
await wiki_packages.restore_package(app, {"id": "target", "weknora_id": "remote"}, "job-test", exported)
for raw in received:
assert raw["content"] == next(p["content"] for p in pages if p["slug"] == raw["slug"])
local = await index.get_page("target", raw["slug"])
assert local["content_hash"] == normalize_wiki_page("target", raw)["content_hash"]
snapshot = await index.load_vector_snapshot("target", "test-fp")
assert len(snapshot["rows"]) == 2
assert all(r["vector_blob"] == blob for r in snapshot["rows"])
class SearchEmbedding:
dimensions = 3
fingerprint = "test-fp"
async def embed_query(self, query):
return [1, 0, 0]
found = await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding())
assert found and all(r["retrieval_mode"] == "vector" for r in found)
SearchEmbedding.fingerprint = "different-model-same-dimension"
assert await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding()) == []
archive.close()
restored_archive.close()
await engine.dispose()
@pytest.mark.asyncio
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
observed = {}
class LocalModel:
@staticmethod
def list_supported_models():
return [{"model": "BAAI/bge-m3"}]
def __init__(self, **kwargs):
observed.update(kwargs)
def embed(self, texts, *, batch_size):
observed["batch_size"] = batch_size
return [[1.0, *([0.0] * 1023)] for _ in texts]
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
client = StrictWikiEmbeddingClient(config)
assert len(await client.embed_query("本地查询")) == 1024
assert observed["model_name"] == "BAAI/bge-m3"
assert observed["providers"] == ["CPUExecutionProvider"]
assert observed["batch_size"] == observed["threads"] == 2
assert client._client is None
@pytest.mark.asyncio
async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypatch, tmp_path):
from starlette.requests import Request
from app.gateway.routers import skill_knowledge
async def admin(_):
return "admin"
class Mappings:
async def get_authorized(self, *args, **kwargs):
return {"name": "测试库", "weknora_id": "remote"}
class Jobs:
async def create_job(self, **kwargs):
return kwargs
app = FastAPI()
app.state.config = None
app.state.llmwiki_store = Mappings()
app.state.skill_knowledge_store = Jobs()
# Deliberately no skill_store ownership row: built-ins live on disk.
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
assert job["skill_names"] == ["builtin"]
assert job["review_mode"] == "off"
def test_short_wiki_summary_and_body_still_have_a_vector_section():
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
assert len(sections) == 1
assert "降低设备温度" in sections[0].embedding_text