feat: improve knowledge sync and entity alignment

This commit is contained in:
吉鑫升 committed 2026-09-04 18:43:12 +08:00
1 parent 7dbac9cc56
commit dea4697ea0
38 files changed
+3742 -634

No files matched your search

@@ -1,15 +1,18 @@
import base64
import json
import sys
from io import BytesIO
from types import SimpleNamespace
import pytest
from fastapi import FastAPI
from fastapi import FastAPI, UploadFile
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.routers import wiki_packages
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import write_package
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
from deerflow.integrations.weknora.local_index import embedding as embedding_module
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
@@ -146,7 +149,7 @@ async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp
@pytest.mark.asyncio
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch, tmp_path):
observed = {}
class LocalModel:
@@ -162,13 +165,115 @@ async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
return [[1.0, *([0.0] * 1023)] for _ in texts]
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
(tmp_path / "onnx").mkdir()
(tmp_path / "onnx" / "model.onnx").touch()
config = LocalWikiEmbeddingConfig(
provider="local",
model="bge-embedding-m3",
dimensions=1024,
batch_size=2,
local_threads=2,
local_model_path=str(tmp_path),
local_process_isolation=False,
)
client = StrictWikiEmbeddingClient(config)
assert len(await client.embed_query("本地查询")) == 1024
assert observed["model_name"] == "BAAI/bge-m3"
assert observed["providers"] == ["CPUExecutionProvider"]
assert observed["local_files_only"] is True
assert observed["specific_model_path"] == str(tmp_path)
assert observed["batch_size"] == observed["threads"] == 2
assert client._client is None
assert client.local_encoder_status()["state"] == "ready"
assert client.local_encoder_status()["model_files_ready"] is True
cold_client = StrictWikiEmbeddingClient(config)
status = cold_client.start_local_encoder()
assert status["state"] == "starting"
assert status["running"] is True
await cold_client._local_warmup_task
assert cold_client.local_encoder_status()["state"] == "ready"
monkeypatch.delenv("DEERFLOW_BGE_M3_MODEL_PATH", raising=False)
missing_path = StrictWikiEmbeddingClient(
LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024),
)
missing_path.start_local_encoder()
await missing_path._local_warmup_task
assert missing_path.local_encoder_status()["state"] == "failed"
assert "目录未配置" in str(missing_path.local_encoder_status()["last_error"])
@pytest.mark.asyncio
async def test_local_encoder_uses_offline_isolated_worker(monkeypatch, tmp_path):
(tmp_path / "onnx").mkdir()
(tmp_path / "onnx" / "model.onnx").touch()
observed: dict = {}
class FakeStdout:
def __init__(self):
self.lines = [json.dumps({"status": "ready"}) + "\n"]
def readline(self):
return self.lines.pop(0) if self.lines else ""
class FakeProcess:
def __init__(self):
self.stdout = FakeStdout()
self.returncode = None
self.stdin = SimpleNamespace(write=self.write, flush=lambda: None)
def write(self, line):
payload = json.loads(line)
if payload.get("command") == "shutdown":
self.returncode = 0
return
self.stdout.lines.append(
json.dumps(
{
"id": payload["id"],
"status": "ok",
"vectors": [[1.0, *([0.0] * 1023)] for _ in payload["texts"]],
}
)
+ "\n"
)
def poll(self):
return self.returncode
def wait(self, timeout=None):
return self.returncode
def terminate(self):
self.returncode = -15
def kill(self):
self.returncode = -9
process = FakeProcess()
def fake_popen(command, **kwargs):
observed.update({"command": command, **kwargs})
return process
monkeypatch.setattr(embedding_module.subprocess, "Popen", fake_popen)
client = StrictWikiEmbeddingClient(
LocalWikiEmbeddingConfig(
provider="local",
model="bge-embedding-m3",
dimensions=1024,
local_model_path=str(tmp_path),
)
)
assert len(await client.embed_query("隔离编码")) == 1024
assert "deerflow.integrations.weknora.local_index.embedding_worker" in observed["command"]
assert observed["env"]["HF_HUB_OFFLINE"] == "1"
assert observed["env"]["PYTHONIOENCODING"] == "utf-8"
assert client.local_encoder_status()["process_isolation"] is True
await client.close()
assert process.poll() == 0
@pytest.mark.asyncio
@@ -178,7 +283,7 @@ async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypat
from app.gateway.routers import skill_knowledge
async def admin(_):
return "admin"
return "admin", True
class Mappings:
async def get_authorized(self, *args, **kwargs):
@@ -193,11 +298,61 @@ async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypat
app.state.llmwiki_store = Mappings()
app.state.skill_knowledge_store = Jobs()
# Deliberately no skill_store ownership row: built-ins live on disk.
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
monkeypatch.setattr(skill_knowledge, "_actor", admin)
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
assert job["skill_names"] == ["builtin"]
@pytest.mark.asyncio
async def test_regular_user_can_distill_visible_skill_only_to_writable_wiki(monkeypatch, tmp_path):
from starlette.requests import Request
from app.gateway.routers import skill_knowledge
async def regular_user(_):
return "user-1", False
class Skills:
async def get_visible(self, name, user_id):
assert user_id == "user-1"
return {"name": name} if name == "visible-skill" else None
async def list_favorite_skill_names(self, user_id):
assert user_id == "user-1"
return []
class Mappings:
async def get_authorized(self, mapping_id, user_id, *, write, is_admin):
assert (mapping_id, user_id, write, is_admin) == ("mapping", "user-1", True, False)
return {"name": "我的 Wiki", "weknora_id": "remote"}
class Jobs:
async def create_job(self, **kwargs):
return kwargs
app = FastAPI()
app.state.config = None
app.state.skill_store = Skills()
app.state.llmwiki_store = Mappings()
app.state.skill_knowledge_store = Jobs()
monkeypatch.setattr(skill_knowledge, "_actor", regular_user)
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
monkeypatch.setattr(
skill_knowledge,
"get_or_new_skill_storage",
lambda **_: SimpleNamespace(get_skills_root_path=lambda: tmp_path),
)
body = skill_knowledge.CreateSyncJobRequest(
skill_names=["visible-skill"],
targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}],
review_mode="required",
)
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
assert job["created_by"] == "user-1"
assert job["review_mode"] == "off"
@@ -205,3 +360,70 @@ def test_short_wiki_summary_and_body_still_have_a_vector_section():
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
assert len(sections) == 1
assert "降低设备温度" in sections[0].embedding_text
@pytest.mark.asyncio
async def test_admin_can_upload_and_activate_offline_encoder(monkeypatch, tmp_path):
from starlette.requests import Request
from app.gateway.routers import llmwiki_index
from deerflow.config.system_settings import SystemSettings
async def admin(_):
return "admin", True
class Embedding:
def __init__(self):
self.model_path = ""
async def reset_local_encoder(self, *, model_path=None):
if model_path is not None:
self.model_path = model_path
def local_encoder_status(self):
return {
"provider": "local",
"model": "bge-embedding-m3",
"state": "idle",
"model_path": self.model_path,
"model_files_ready": bool(self.model_path),
}
saved = []
monkeypatch.setattr(llmwiki_index, "_actor", admin)
monkeypatch.setattr(llmwiki_index, "runtime_home", lambda: tmp_path)
monkeypatch.setattr(llmwiki_index, "load_system_settings", SystemSettings)
monkeypatch.setattr(llmwiki_index, "save_system_settings", lambda value: saved.append(value) or value)
embedding = Embedding()
app = FastAPI()
app.state.config = SimpleNamespace(
llmwiki=SimpleNamespace(
local_wiki_index=SimpleNamespace(embedding=SimpleNamespace(provider="local"))
)
)
app.state.llmwiki_embedding = embedding
request = Request({"type": "http", "headers": [], "app": app})
files = [
UploadFile(filename="model.onnx", file=BytesIO(b"onnx")),
UploadFile(filename="config.json", file=BytesIO(b"{}")),
]
result = await llmwiki_index.upload_local_encoder(
request,
files=files,
paths=["onnx/model.onnx", "config.json"],
)
active = tmp_path / "models" / "bge-m3"
assert (active / "onnx" / "model.onnx").read_bytes() == b"onnx"
assert result["uploaded_files"] == 2
assert saved[-1].local_wiki_encoder.model_path == str(active.resolve())
assert embedding.model_path == str(active.resolve())
def test_encoder_upload_rejects_parent_traversal():
from app.gateway.routers import llmwiki_index
with pytest.raises(Exception) as exc_info:
llmwiki_index._safe_relative_upload_path("../model.onnx")
assert getattr(exc_info.value, "status_code", None) == 422