feat: preserve Wiki vectors across knowledge transfers and add local BGE-M3
Unify Wiki reading, preserve metadata and graph links, add asynchronous file-backed transfers, and reuse compatible vectors. Includes regression coverage and live acceptance notes.
This commit is contained in:
1 parent
49d9ced04f
commit
60e2c72514
38 files changed
+2536
-581
No files matched your search
@@ -13,6 +13,7 @@ from deerflow.persistence.base import Base
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
dimensions = 3
|
||||
fingerprint = "test-bge-m3"
|
||||
|
||||
async def embed_query(self, _query: str) -> list[float]:
|
||||
return [1.0, 0.0, 0.0]
|
||||
@@ -154,6 +155,7 @@ async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
"section_index": 0,
|
||||
"section_content": "张三负责向量化检索。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_fingerprint": "test-bge-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [1.0, 0.0, 0.0],
|
||||
},
|
||||
@@ -162,6 +164,7 @@ async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
"section_index": 1,
|
||||
"section_content": "无关片段。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_fingerprint": "test-bge-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [0.0, 1.0, 0.0],
|
||||
},
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Ordinary knowledge-base editing and remote-deletion reconciliation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import FastAPI
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.persistence.llmwiki import MemoryLlmWikiStore
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def knowledge_api(monkeypatch: pytest.MonkeyPatch):
|
||||
from app.gateway.routers import llmwiki
|
||||
|
||||
store = MemoryLlmWikiStore()
|
||||
row = await store.create_mapping(weknora_id="remote-one", owner_user_id="owner", name="原名称", description="原描述", kb_type="document")
|
||||
await store.request_publish(row["id"], "owner", is_admin=False)
|
||||
state = SimpleNamespace(
|
||||
store=store,
|
||||
mapping_id=row["id"],
|
||||
remote={"remote-one": {"id": "remote-one", "name": "原名称", "description": "原描述", "knowledge_count": 3}},
|
||||
user_id="owner",
|
||||
is_admin=False,
|
||||
upstream_status=200,
|
||||
writes=[],
|
||||
)
|
||||
|
||||
async def upstream(request: httpx.Request) -> httpx.Response:
|
||||
if state.upstream_status != 200:
|
||||
return httpx.Response(state.upstream_status, json={"message": "知识服务暂不可用"})
|
||||
if request.url.path == "/api/v1/knowledge-bases":
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(200, json={"data": list(state.remote.values())})
|
||||
remote_id = request.url.path.rsplit("/", 1)[-1]
|
||||
if remote_id not in state.remote:
|
||||
return httpx.Response(404)
|
||||
if request.method == "PUT":
|
||||
changes = json.loads(request.content)
|
||||
state.writes.append(changes)
|
||||
state.remote[remote_id].update(changes)
|
||||
else:
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(200, json={"data": state.remote[remote_id]})
|
||||
|
||||
async def actor(_: object) -> tuple[str, bool]:
|
||||
return state.user_id, state.is_admin
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(llmwiki.router)
|
||||
monkeypatch.setattr(llmwiki, "_actor", actor)
|
||||
monkeypatch.setattr(llmwiki, "_store", lambda _: store)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as remote_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.test", http_client=remote_client)
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: client)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as api:
|
||||
state.api = api
|
||||
yield state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scope", ["personal", "public", "all"])
|
||||
@pytest.mark.parametrize("is_admin", [False, True])
|
||||
async def test_lists_hide_remote_deleted_bases_without_deleting_mappings(knowledge_api, scope: str, is_admin: bool) -> None:
|
||||
state = knowledge_api
|
||||
state.is_admin = is_admin
|
||||
live = await state.store.create_mapping(weknora_id="remote-live", owner_user_id="owner", name="保留知识库", description="", kb_type="document")
|
||||
await state.store.request_publish(live["id"], "owner", is_admin=False)
|
||||
state.remote["remote-live"] = {"id": "remote-live", "name": "保留知识库"}
|
||||
deleted = state.remote.pop("remote-one")
|
||||
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [live["id"]]
|
||||
assert await state.store.get_by_weknora_id("remote-one") is not None
|
||||
|
||||
state.remote.clear()
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
assert response.json()["knowledge_bases"] == []
|
||||
|
||||
state.remote["remote-one"] = deleted
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [502, 504])
|
||||
async def test_list_upstream_failure_is_not_treated_as_remote_deletion(knowledge_api, status: int) -> None:
|
||||
state = knowledge_api
|
||||
state.upstream_status = status
|
||||
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
|
||||
assert response.status_code >= 500
|
||||
assert await state.store.get_by_weknora_id("remote-one") is not None
|
||||
state.upstream_status = 200
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "expected_name", "expected_description"),
|
||||
[
|
||||
({"name": " 新名称 ", "description": "新描述"}, "新名称", "新描述"),
|
||||
({"name": "仅修改名称"}, "仅修改名称", "原描述"),
|
||||
({"description": "仅修改描述"}, "原名称", "仅修改描述"),
|
||||
({"name": "清空描述", "description": ""}, "清空描述", ""),
|
||||
],
|
||||
)
|
||||
async def test_edit_synchronizes_remote_and_local_metadata(knowledge_api, changes, expected_name: str, expected_description: str) -> None:
|
||||
state = knowledge_api
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json=changes)
|
||||
|
||||
assert response.status_code == 200
|
||||
item = response.json()
|
||||
assert (item["name"], item["description"]) == (expected_name, expected_description)
|
||||
assert item["remote_status"] == "available"
|
||||
assert item["knowledge_count"] == 3
|
||||
assert item["publication_status"] == "published"
|
||||
local = await state.store.get_by_weknora_id("remote-one")
|
||||
assert (local["name"], local["description"]) == (expected_name, expected_description)
|
||||
assert state.writes == [{**changes, "name": expected_name}]
|
||||
listing = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
assert listing.json()["knowledge_bases"][0]["name"] == expected_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["", " \t ", "x" * 256, "对话沉淀", " 对话沉淀 "])
|
||||
async def test_edit_rejects_invalid_or_reserved_names(knowledge_api, name: str) -> None:
|
||||
state = knowledge_api
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": name})
|
||||
|
||||
assert response.status_code in {409, 422}
|
||||
assert state.writes == []
|
||||
assert (await state.store.get_by_weknora_id("remote-one"))["name"] == "原名称"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("is_admin", [False, True])
|
||||
async def test_only_owner_or_admin_can_edit_a_public_base(knowledge_api, is_admin: bool) -> None:
|
||||
state = knowledge_api
|
||||
state.user_id = "other-user"
|
||||
state.is_admin = is_admin
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
|
||||
|
||||
assert response.status_code == (200 if is_admin else 404)
|
||||
assert len(state.writes) == (1 if is_admin else 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_cannot_edit_system_conversation_deposit(knowledge_api) -> None:
|
||||
state = knowledge_api
|
||||
state.is_admin = True
|
||||
await state.store.adopt_conversation_deposit_mapping(state.mapping_id, name="对话沉淀", description="系统知识库")
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
|
||||
|
||||
assert response.status_code == 403
|
||||
assert state.writes == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [404, 502, 504])
|
||||
async def test_failed_remote_edit_leaves_local_metadata_unchanged(knowledge_api, status: int) -> None:
|
||||
state = knowledge_api
|
||||
state.upstream_status = status
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称", "description": "新描述"})
|
||||
|
||||
if status == 404:
|
||||
assert response.status_code == 404
|
||||
else:
|
||||
assert response.status_code >= 500
|
||||
local = await state.store.get_by_weknora_id("remote-one")
|
||||
assert (local["name"], local["description"]) == ("原名称", "原描述")
|
||||
@@ -0,0 +1,207 @@
|
||||
import base64
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.routers import wiki_packages
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import write_package
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp_path, monkeypatch):
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
blob, _ = encode_vector([0.3, 0.5, 0.7])
|
||||
pages = [
|
||||
{
|
||||
"slug": "concept/cooling",
|
||||
"title": "散热",
|
||||
"page_type": "concept",
|
||||
"content": "散热管理用于降低设备温度。参见 [[entity/fan]]",
|
||||
"summary": "设备散热",
|
||||
"tags": ["设备", "维护"],
|
||||
"category_path": ["运维", "硬件"],
|
||||
"aliases": ["降温"],
|
||||
"out_links": ["entity/fan"],
|
||||
"in_links": [],
|
||||
},
|
||||
{"slug": "entity/fan", "title": "风扇", "page_type": "entity", "content": "风扇通过气流帮助散热,参见 [[concept/cooling]]。", "aliases": [], "out_links": ["concept/cooling"], "in_links": ["concept/cooling"]},
|
||||
]
|
||||
vectors = [
|
||||
{
|
||||
"wiki_slug": p["slug"],
|
||||
"section_index": 0,
|
||||
"section_content": p["content"],
|
||||
"content_hash": "test",
|
||||
"embedding_model": "test-model",
|
||||
"embedding_fingerprint": "test-fp",
|
||||
"embedding_dimensions": 3,
|
||||
"vector_blob_base64": base64.b64encode(blob).decode(),
|
||||
}
|
||||
for p in pages
|
||||
]
|
||||
original = tmp_path / "original.jsonl"
|
||||
entities = [{"id": p["slug"], "name": p["title"], "type": p["page_type"], "source_wiki_slug": p["slug"]} for p in pages]
|
||||
relations = [{"source_entity_id": pages[0]["slug"], "target_entity_id": pages[1]["slug"], "source_wiki_slug": pages[0]["slug"], "predicate": "references"}]
|
||||
write_package(original, [
|
||||
("source", {"provider": "weknora", "package_version": 1}),
|
||||
*[("wiki_page", p) for p in pages],
|
||||
*[("vector", v) for v in vectors],
|
||||
*[("entity", e) for e in entities],
|
||||
*[("relation", r) for r in relations],
|
||||
("chunk", {"id": "raw-chunk", "content": "原始文档分段不是 Wiki 页面"}),
|
||||
])
|
||||
archive = PackageArchive(original)
|
||||
base = await store.initialize(created_by="admin")
|
||||
job = await store.import_package(package=archive, source_type="weknora", source_key="source", source_name="来源", trigger="test", created_by="admin")
|
||||
assert job["counts"]["vectors"] == 2
|
||||
assert job["counts"]["wiki_pages"] == 2
|
||||
assert job["counts"]["chunk_pages"] == 1
|
||||
imported = await store.get_page(pages[0]["slug"])
|
||||
for field in ("category_path", "tags", "aliases", "out_links"):
|
||||
assert imported[field] == pages[0][field]
|
||||
assert imported["content_markdown"] == pages[0]["content"]
|
||||
# The global base may receive the same Wiki through multiple libraries.
|
||||
# Only the current article's import supplies vectors; older contributions
|
||||
# remain auditable but must not duplicate/stale the restored index.
|
||||
await store.import_package(package=archive, source_type="weknora", source_key="second-source", source_name="第二来源", trigger="test", created_by="admin")
|
||||
exported = tmp_path / "export.jsonl"
|
||||
write_package(exported, [record async for record in store.iter_export_records(base["id"])])
|
||||
restored_archive = PackageArchive(exported)
|
||||
assert {v["vector_blob_base64"] for v in restored_archive["vectors"]} == {base64.b64encode(blob).decode()}
|
||||
assert sorted((v["wiki_slug"], v["section_index"]) for v in restored_archive["vectors"]) == sorted((v["wiki_slug"], v["section_index"]) for v in vectors)
|
||||
assert all(r["source_wiki_slug"] in {p["slug"] for p in pages} for r in restored_archive["relations"])
|
||||
index = MemoryLlmWikiIndexStore()
|
||||
received = []
|
||||
|
||||
class Remote:
|
||||
async def create_wiki_folder(self, kb, *, name, parent_id=""):
|
||||
return {"id": f"{parent_id}/{name}"}
|
||||
|
||||
async def create_wiki_page(self, kb, page):
|
||||
received.append(page)
|
||||
return page
|
||||
|
||||
async def rebuild_wiki_links(self, kb):
|
||||
pass
|
||||
|
||||
class Embedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
raise AssertionError("Existing package must not be encoded")
|
||||
|
||||
app = FastAPI()
|
||||
app.state.llmwiki_index_store = index
|
||||
app.state.llmwiki_embedding = Embedding()
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.config = None
|
||||
monkeypatch.setattr(wiki_packages, "get_resolved_llmwiki_runtime", lambda c: None)
|
||||
monkeypatch.setattr(wiki_packages, "build_weknora_client", lambda c: Remote())
|
||||
monkeypatch.setattr(wiki_packages, "_package_root", lambda: tmp_path)
|
||||
from app.gateway import knowledge_transfer
|
||||
|
||||
async def sync(app, mapping):
|
||||
return {"status": "completed"}
|
||||
|
||||
monkeypatch.setattr(knowledge_transfer, "sync_to_global", sync)
|
||||
await wiki_packages.restore_package(app, {"id": "target", "weknora_id": "remote"}, "job-test", exported)
|
||||
for raw in received:
|
||||
assert raw["content"] == next(p["content"] for p in pages if p["slug"] == raw["slug"])
|
||||
local = await index.get_page("target", raw["slug"])
|
||||
assert local["content_hash"] == normalize_wiki_page("target", raw)["content_hash"]
|
||||
snapshot = await index.load_vector_snapshot("target", "test-fp")
|
||||
assert len(snapshot["rows"]) == 2
|
||||
assert all(r["vector_blob"] == blob for r in snapshot["rows"])
|
||||
|
||||
class SearchEmbedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
return [1, 0, 0]
|
||||
|
||||
found = await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding())
|
||||
assert found and all(r["retrieval_mode"] == "vector" for r in found)
|
||||
SearchEmbedding.fingerprint = "different-model-same-dimension"
|
||||
assert await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding()) == []
|
||||
archive.close()
|
||||
restored_archive.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
class LocalModel:
|
||||
@staticmethod
|
||||
def list_supported_models():
|
||||
return [{"model": "BAAI/bge-m3"}]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
|
||||
def embed(self, texts, *, batch_size):
|
||||
observed["batch_size"] = batch_size
|
||||
return [[1.0, *([0.0] * 1023)] for _ in texts]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
|
||||
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
|
||||
client = StrictWikiEmbeddingClient(config)
|
||||
assert len(await client.embed_query("本地查询")) == 1024
|
||||
assert observed["model_name"] == "BAAI/bge-m3"
|
||||
assert observed["providers"] == ["CPUExecutionProvider"]
|
||||
assert observed["batch_size"] == observed["threads"] == 2
|
||||
assert client._client is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def admin(_):
|
||||
return "admin"
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, *args, **kwargs):
|
||||
return {"name": "测试库", "weknora_id": "remote"}
|
||||
|
||||
class Jobs:
|
||||
async def create_job(self, **kwargs):
|
||||
return kwargs
|
||||
|
||||
app = FastAPI()
|
||||
app.state.config = None
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
# Deliberately no skill_store ownership row: built-ins live on disk.
|
||||
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
assert job["skill_names"] == ["builtin"]
|
||||
assert job["review_mode"] == "off"
|
||||
|
||||
|
||||
def test_short_wiki_summary_and_body_still_have_a_vector_section():
|
||||
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
|
||||
assert len(sections) == 1
|
||||
assert "降低设备温度" in sections[0].embedding_text
|
||||
Reference in new issue
Block a user