feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3

Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
吉鑫升 committed 2026-09-03 19:21:10 +08:00
1 parent 49d9ced04f
commit 60e2c72514
38 files changed
+2536 -581

No files matched your search

@@ -13,6 +13,7 @@ from deerflow.persistence.base import Base
class FakeEmbeddingClient:
dimensions = 3
fingerprint = "test-bge-m3"
async def embed_query(self, _query: str) -> list[float]:
return [1.0, 0.0, 0.0]
@@ -154,6 +155,7 @@ async def test_imported_vectors_are_reused_for_assistant_search(
"section_index": 0,
"section_content": "张三负责向量化检索。",
"embedding_model": "bge-embedding-m3",
"embedding_fingerprint": "test-bge-m3",
"embedding_dimensions": 3,
"vector": [1.0, 0.0, 0.0],
},
@@ -162,6 +164,7 @@ async def test_imported_vectors_are_reused_for_assistant_search(
"section_index": 1,
"section_content": "无关片段。",
"embedding_model": "bge-embedding-m3",
"embedding_fingerprint": "test-bge-m3",
"embedding_dimensions": 3,
"vector": [0.0, 1.0, 0.0],
},
@@ -0,0 +1,180 @@
"""Ordinary knowledge-base editing and remote-deletion reconciliation."""
from __future__ import annotations
import json
from types import SimpleNamespace
import httpx
import pytest
import pytest_asyncio
from fastapi import FastAPI
from deerflow.integrations.weknora.client import WeKnoraClient
from deerflow.persistence.llmwiki import MemoryLlmWikiStore
@pytest_asyncio.fixture
async def knowledge_api(monkeypatch: pytest.MonkeyPatch):
from app.gateway.routers import llmwiki
store = MemoryLlmWikiStore()
row = await store.create_mapping(weknora_id="remote-one", owner_user_id="owner", name="原名称", description="原描述", kb_type="document")
await store.request_publish(row["id"], "owner", is_admin=False)
state = SimpleNamespace(
store=store,
mapping_id=row["id"],
remote={"remote-one": {"id": "remote-one", "name": "原名称", "description": "原描述", "knowledge_count": 3}},
user_id="owner",
is_admin=False,
upstream_status=200,
writes=[],
)
async def upstream(request: httpx.Request) -> httpx.Response:
if state.upstream_status != 200:
return httpx.Response(state.upstream_status, json={"message": "知识服务暂不可用"})
if request.url.path == "/api/v1/knowledge-bases":
assert request.method == "GET"
return httpx.Response(200, json={"data": list(state.remote.values())})
remote_id = request.url.path.rsplit("/", 1)[-1]
if remote_id not in state.remote:
return httpx.Response(404)
if request.method == "PUT":
changes = json.loads(request.content)
state.writes.append(changes)
state.remote[remote_id].update(changes)
else:
assert request.method == "GET"
return httpx.Response(200, json={"data": state.remote[remote_id]})
async def actor(_: object) -> tuple[str, bool]:
return state.user_id, state.is_admin
app = FastAPI()
app.include_router(llmwiki.router)
monkeypatch.setattr(llmwiki, "_actor", actor)
monkeypatch.setattr(llmwiki, "_store", lambda _: store)
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as remote_client:
client = WeKnoraClient(base_url="http://weknora.test", http_client=remote_client)
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: client)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as api:
state.api = api
yield state
@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ["personal", "public", "all"])
@pytest.mark.parametrize("is_admin", [False, True])
async def test_lists_hide_remote_deleted_bases_without_deleting_mappings(knowledge_api, scope: str, is_admin: bool) -> None:
state = knowledge_api
state.is_admin = is_admin
live = await state.store.create_mapping(weknora_id="remote-live", owner_user_id="owner", name="保留知识库", description="", kb_type="document")
await state.store.request_publish(live["id"], "owner", is_admin=False)
state.remote["remote-live"] = {"id": "remote-live", "name": "保留知识库"}
deleted = state.remote.pop("remote-one")
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
assert response.status_code == 200
assert [item["id"] for item in response.json()["knowledge_bases"]] == [live["id"]]
assert await state.store.get_by_weknora_id("remote-one") is not None
state.remote.clear()
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
assert response.json()["knowledge_bases"] == []
state.remote["remote-one"] = deleted
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [502, 504])
async def test_list_upstream_failure_is_not_treated_as_remote_deletion(knowledge_api, status: int) -> None:
state = knowledge_api
state.upstream_status = status
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
assert response.status_code >= 500
assert await state.store.get_by_weknora_id("remote-one") is not None
state.upstream_status = 200
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("changes", "expected_name", "expected_description"),
[
({"name": " 新名称 ", "description": "新描述"}, "新名称", "新描述"),
({"name": "仅修改名称"}, "仅修改名称", "原描述"),
({"description": "仅修改描述"}, "原名称", "仅修改描述"),
({"name": "清空描述", "description": ""}, "清空描述", ""),
],
)
async def test_edit_synchronizes_remote_and_local_metadata(knowledge_api, changes, expected_name: str, expected_description: str) -> None:
state = knowledge_api
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json=changes)
assert response.status_code == 200
item = response.json()
assert (item["name"], item["description"]) == (expected_name, expected_description)
assert item["remote_status"] == "available"
assert item["knowledge_count"] == 3
assert item["publication_status"] == "published"
local = await state.store.get_by_weknora_id("remote-one")
assert (local["name"], local["description"]) == (expected_name, expected_description)
assert state.writes == [{**changes, "name": expected_name}]
listing = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
assert listing.json()["knowledge_bases"][0]["name"] == expected_name
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["", " \t ", "x" * 256, "对话沉淀", " 对话沉淀 "])
async def test_edit_rejects_invalid_or_reserved_names(knowledge_api, name: str) -> None:
state = knowledge_api
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": name})
assert response.status_code in {409, 422}
assert state.writes == []
assert (await state.store.get_by_weknora_id("remote-one"))["name"] == "原名称"
@pytest.mark.asyncio
@pytest.mark.parametrize("is_admin", [False, True])
async def test_only_owner_or_admin_can_edit_a_public_base(knowledge_api, is_admin: bool) -> None:
state = knowledge_api
state.user_id = "other-user"
state.is_admin = is_admin
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
assert response.status_code == (200 if is_admin else 404)
assert len(state.writes) == (1 if is_admin else 0)
@pytest.mark.asyncio
async def test_admin_cannot_edit_system_conversation_deposit(knowledge_api) -> None:
state = knowledge_api
state.is_admin = True
await state.store.adopt_conversation_deposit_mapping(state.mapping_id, name="对话沉淀", description="系统知识库")
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
assert response.status_code == 403
assert state.writes == []
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [404, 502, 504])
async def test_failed_remote_edit_leaves_local_metadata_unchanged(knowledge_api, status: int) -> None:
state = knowledge_api
state.upstream_status = status
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称", "description": "新描述"})
if status == 404:
assert response.status_code == 404
else:
assert response.status_code >= 500
local = await state.store.get_by_weknora_id("remote-one")
assert (local["name"], local["description"]) == ("原名称", "原描述")
@@ -0,0 +1,207 @@
import base64
import sys
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers import wiki_packages
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import write_package
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.base import Base
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
@pytest.mark.asyncio
async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp_path, monkeypatch):
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
blob, _ = encode_vector([0.3, 0.5, 0.7])
pages = [
{
"slug": "concept/cooling",
"title": "散热",
"page_type": "concept",
"content": "散热管理用于降低设备温度。参见 [[entity/fan]]",
"summary": "设备散热",
"tags": ["设备", "维护"],
"category_path": ["运维", "硬件"],
"aliases": ["降温"],
"out_links": ["entity/fan"],
"in_links": [],
},
{"slug": "entity/fan", "title": "风扇", "page_type": "entity", "content": "风扇通过气流帮助散热,参见 [[concept/cooling]]。", "aliases": [], "out_links": ["concept/cooling"], "in_links": ["concept/cooling"]},
]
vectors = [
{
"wiki_slug": p["slug"],
"section_index": 0,
"section_content": p["content"],
"content_hash": "test",
"embedding_model": "test-model",
"embedding_fingerprint": "test-fp",
"embedding_dimensions": 3,
"vector_blob_base64": base64.b64encode(blob).decode(),
}
for p in pages
]
original = tmp_path / "original.jsonl"
entities = [{"id": p["slug"], "name": p["title"], "type": p["page_type"], "source_wiki_slug": p["slug"]} for p in pages]
relations = [{"source_entity_id": pages[0]["slug"], "target_entity_id": pages[1]["slug"], "source_wiki_slug": pages[0]["slug"], "predicate": "references"}]
write_package(original, [
("source", {"provider": "weknora", "package_version": 1}),
*[("wiki_page", p) for p in pages],
*[("vector", v) for v in vectors],
*[("entity", e) for e in entities],
*[("relation", r) for r in relations],
("chunk", {"id": "raw-chunk", "content": "原始文档分段不是 Wiki 页面"}),
])
archive = PackageArchive(original)
base = await store.initialize(created_by="admin")
job = await store.import_package(package=archive, source_type="weknora", source_key="source", source_name="来源", trigger="test", created_by="admin")
assert job["counts"]["vectors"] == 2
assert job["counts"]["wiki_pages"] == 2
assert job["counts"]["chunk_pages"] == 1
imported = await store.get_page(pages[0]["slug"])
for field in ("category_path", "tags", "aliases", "out_links"):
assert imported[field] == pages[0][field]
assert imported["content_markdown"] == pages[0]["content"]
# The global base may receive the same Wiki through multiple libraries.
# Only the current article's import supplies vectors; older contributions
# remain auditable but must not duplicate/stale the restored index.
await store.import_package(package=archive, source_type="weknora", source_key="second-source", source_name="第二来源", trigger="test", created_by="admin")
exported = tmp_path / "export.jsonl"
write_package(exported, [record async for record in store.iter_export_records(base["id"])])
restored_archive = PackageArchive(exported)
assert {v["vector_blob_base64"] for v in restored_archive["vectors"]} == {base64.b64encode(blob).decode()}
assert sorted((v["wiki_slug"], v["section_index"]) for v in restored_archive["vectors"]) == sorted((v["wiki_slug"], v["section_index"]) for v in vectors)
assert all(r["source_wiki_slug"] in {p["slug"] for p in pages} for r in restored_archive["relations"])
index = MemoryLlmWikiIndexStore()
received = []
class Remote:
async def create_wiki_folder(self, kb, *, name, parent_id=""):
return {"id": f"{parent_id}/{name}"}
async def create_wiki_page(self, kb, page):
received.append(page)
return page
async def rebuild_wiki_links(self, kb):
pass
class Embedding:
dimensions = 3
fingerprint = "test-fp"
async def embed_query(self, query):
raise AssertionError("Existing package must not be encoded")
app = FastAPI()
app.state.llmwiki_index_store = index
app.state.llmwiki_embedding = Embedding()
app.state.llmwiki_vector_cache = None
app.state.config = None
monkeypatch.setattr(wiki_packages, "get_resolved_llmwiki_runtime", lambda c: None)
monkeypatch.setattr(wiki_packages, "build_weknora_client", lambda c: Remote())
monkeypatch.setattr(wiki_packages, "_package_root", lambda: tmp_path)
from app.gateway import knowledge_transfer
async def sync(app, mapping):
return {"status": "completed"}
monkeypatch.setattr(knowledge_transfer, "sync_to_global", sync)
await wiki_packages.restore_package(app, {"id": "target", "weknora_id": "remote"}, "job-test", exported)
for raw in received:
assert raw["content"] == next(p["content"] for p in pages if p["slug"] == raw["slug"])
local = await index.get_page("target", raw["slug"])
assert local["content_hash"] == normalize_wiki_page("target", raw)["content_hash"]
snapshot = await index.load_vector_snapshot("target", "test-fp")
assert len(snapshot["rows"]) == 2
assert all(r["vector_blob"] == blob for r in snapshot["rows"])
class SearchEmbedding:
dimensions = 3
fingerprint = "test-fp"
async def embed_query(self, query):
return [1, 0, 0]
found = await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding())
assert found and all(r["retrieval_mode"] == "vector" for r in found)
SearchEmbedding.fingerprint = "different-model-same-dimension"
assert await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding()) == []
archive.close()
restored_archive.close()
await engine.dispose()
@pytest.mark.asyncio
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
observed = {}
class LocalModel:
@staticmethod
def list_supported_models():
return [{"model": "BAAI/bge-m3"}]
def __init__(self, **kwargs):
observed.update(kwargs)
def embed(self, texts, *, batch_size):
observed["batch_size"] = batch_size
return [[1.0, *([0.0] * 1023)] for _ in texts]
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
client = StrictWikiEmbeddingClient(config)
assert len(await client.embed_query("本地查询")) == 1024
assert observed["model_name"] == "BAAI/bge-m3"
assert observed["providers"] == ["CPUExecutionProvider"]
assert observed["batch_size"] == observed["threads"] == 2
assert client._client is None
@pytest.mark.asyncio
async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypatch, tmp_path):
from starlette.requests import Request
from app.gateway.routers import skill_knowledge
async def admin(_):
return "admin"
class Mappings:
async def get_authorized(self, *args, **kwargs):
return {"name": "测试库", "weknora_id": "remote"}
class Jobs:
async def create_job(self, **kwargs):
return kwargs
app = FastAPI()
app.state.config = None
app.state.llmwiki_store = Mappings()
app.state.skill_knowledge_store = Jobs()
# Deliberately no skill_store ownership row: built-ins live on disk.
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
assert job["skill_names"] == ["builtin"]
assert job["review_mode"] == "off"
def test_short_wiki_summary_and_body_still_have_a_vector_section():
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
assert len(sections) == 1
assert "降低设备温度" in sections[0].embedding_text