feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
@@ -0,0 +1,207 @@
|
||||
import base64
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.routers import wiki_packages
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import write_package
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp_path, monkeypatch):
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
blob, _ = encode_vector([0.3, 0.5, 0.7])
|
||||
pages = [
|
||||
{
|
||||
"slug": "concept/cooling",
|
||||
"title": "散热",
|
||||
"page_type": "concept",
|
||||
"content": "散热管理用于降低设备温度。参见 [[entity/fan]]",
|
||||
"summary": "设备散热",
|
||||
"tags": ["设备", "维护"],
|
||||
"category_path": ["运维", "硬件"],
|
||||
"aliases": ["降温"],
|
||||
"out_links": ["entity/fan"],
|
||||
"in_links": [],
|
||||
},
|
||||
{"slug": "entity/fan", "title": "风扇", "page_type": "entity", "content": "风扇通过气流帮助散热,参见 [[concept/cooling]]。", "aliases": [], "out_links": ["concept/cooling"], "in_links": ["concept/cooling"]},
|
||||
]
|
||||
vectors = [
|
||||
{
|
||||
"wiki_slug": p["slug"],
|
||||
"section_index": 0,
|
||||
"section_content": p["content"],
|
||||
"content_hash": "test",
|
||||
"embedding_model": "test-model",
|
||||
"embedding_fingerprint": "test-fp",
|
||||
"embedding_dimensions": 3,
|
||||
"vector_blob_base64": base64.b64encode(blob).decode(),
|
||||
}
|
||||
for p in pages
|
||||
]
|
||||
original = tmp_path / "original.jsonl"
|
||||
entities = [{"id": p["slug"], "name": p["title"], "type": p["page_type"], "source_wiki_slug": p["slug"]} for p in pages]
|
||||
relations = [{"source_entity_id": pages[0]["slug"], "target_entity_id": pages[1]["slug"], "source_wiki_slug": pages[0]["slug"], "predicate": "references"}]
|
||||
write_package(original, [
|
||||
("source", {"provider": "weknora", "package_version": 1}),
|
||||
*[("wiki_page", p) for p in pages],
|
||||
*[("vector", v) for v in vectors],
|
||||
*[("entity", e) for e in entities],
|
||||
*[("relation", r) for r in relations],
|
||||
("chunk", {"id": "raw-chunk", "content": "原始文档分段不是 Wiki 页面"}),
|
||||
])
|
||||
archive = PackageArchive(original)
|
||||
base = await store.initialize(created_by="admin")
|
||||
job = await store.import_package(package=archive, source_type="weknora", source_key="source", source_name="来源", trigger="test", created_by="admin")
|
||||
assert job["counts"]["vectors"] == 2
|
||||
assert job["counts"]["wiki_pages"] == 2
|
||||
assert job["counts"]["chunk_pages"] == 1
|
||||
imported = await store.get_page(pages[0]["slug"])
|
||||
for field in ("category_path", "tags", "aliases", "out_links"):
|
||||
assert imported[field] == pages[0][field]
|
||||
assert imported["content_markdown"] == pages[0]["content"]
|
||||
# The global base may receive the same Wiki through multiple libraries.
|
||||
# Only the current article's import supplies vectors; older contributions
|
||||
# remain auditable but must not duplicate/stale the restored index.
|
||||
await store.import_package(package=archive, source_type="weknora", source_key="second-source", source_name="第二来源", trigger="test", created_by="admin")
|
||||
exported = tmp_path / "export.jsonl"
|
||||
write_package(exported, [record async for record in store.iter_export_records(base["id"])])
|
||||
restored_archive = PackageArchive(exported)
|
||||
assert {v["vector_blob_base64"] for v in restored_archive["vectors"]} == {base64.b64encode(blob).decode()}
|
||||
assert sorted((v["wiki_slug"], v["section_index"]) for v in restored_archive["vectors"]) == sorted((v["wiki_slug"], v["section_index"]) for v in vectors)
|
||||
assert all(r["source_wiki_slug"] in {p["slug"] for p in pages} for r in restored_archive["relations"])
|
||||
index = MemoryLlmWikiIndexStore()
|
||||
received = []
|
||||
|
||||
class Remote:
|
||||
async def create_wiki_folder(self, kb, *, name, parent_id=""):
|
||||
return {"id": f"{parent_id}/{name}"}
|
||||
|
||||
async def create_wiki_page(self, kb, page):
|
||||
received.append(page)
|
||||
return page
|
||||
|
||||
async def rebuild_wiki_links(self, kb):
|
||||
pass
|
||||
|
||||
class Embedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
raise AssertionError("Existing package must not be encoded")
|
||||
|
||||
app = FastAPI()
|
||||
app.state.llmwiki_index_store = index
|
||||
app.state.llmwiki_embedding = Embedding()
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.config = None
|
||||
monkeypatch.setattr(wiki_packages, "get_resolved_llmwiki_runtime", lambda c: None)
|
||||
monkeypatch.setattr(wiki_packages, "build_weknora_client", lambda c: Remote())
|
||||
monkeypatch.setattr(wiki_packages, "_package_root", lambda: tmp_path)
|
||||
from app.gateway import knowledge_transfer
|
||||
|
||||
async def sync(app, mapping):
|
||||
return {"status": "completed"}
|
||||
|
||||
monkeypatch.setattr(knowledge_transfer, "sync_to_global", sync)
|
||||
await wiki_packages.restore_package(app, {"id": "target", "weknora_id": "remote"}, "job-test", exported)
|
||||
for raw in received:
|
||||
assert raw["content"] == next(p["content"] for p in pages if p["slug"] == raw["slug"])
|
||||
local = await index.get_page("target", raw["slug"])
|
||||
assert local["content_hash"] == normalize_wiki_page("target", raw)["content_hash"]
|
||||
snapshot = await index.load_vector_snapshot("target", "test-fp")
|
||||
assert len(snapshot["rows"]) == 2
|
||||
assert all(r["vector_blob"] == blob for r in snapshot["rows"])
|
||||
|
||||
class SearchEmbedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
return [1, 0, 0]
|
||||
|
||||
found = await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding())
|
||||
assert found and all(r["retrieval_mode"] == "vector" for r in found)
|
||||
SearchEmbedding.fingerprint = "different-model-same-dimension"
|
||||
assert await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding()) == []
|
||||
archive.close()
|
||||
restored_archive.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
class LocalModel:
|
||||
@staticmethod
|
||||
def list_supported_models():
|
||||
return [{"model": "BAAI/bge-m3"}]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
|
||||
def embed(self, texts, *, batch_size):
|
||||
observed["batch_size"] = batch_size
|
||||
return [[1.0, *([0.0] * 1023)] for _ in texts]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
|
||||
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
|
||||
client = StrictWikiEmbeddingClient(config)
|
||||
assert len(await client.embed_query("本地查询")) == 1024
|
||||
assert observed["model_name"] == "BAAI/bge-m3"
|
||||
assert observed["providers"] == ["CPUExecutionProvider"]
|
||||
assert observed["batch_size"] == observed["threads"] == 2
|
||||
assert client._client is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def admin(_):
|
||||
return "admin"
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, *args, **kwargs):
|
||||
return {"name": "测试库", "weknora_id": "remote"}
|
||||
|
||||
class Jobs:
|
||||
async def create_job(self, **kwargs):
|
||||
return kwargs
|
||||
|
||||
app = FastAPI()
|
||||
app.state.config = None
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
# Deliberately no skill_store ownership row: built-ins live on disk.
|
||||
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
assert job["skill_names"] == ["builtin"]
|
||||
assert job["review_mode"] == "off"
|
||||
|
||||
|
||||
def test_short_wiki_summary_and_body_still_have_a_vector_section():
|
||||
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
|
||||
assert len(sections) == 1
|
||||
assert "降低设备温度" in sections[0].embedding_text
|
||||
Reference in new issue
Block a user