feat: improve knowledge sync and entity alignment
This commit is contained in:
1 parent
7dbac9cc56
commit
dea4697ea0
38 files changed
+3742
-634
No files matched your search
@@ -3,6 +3,7 @@ import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.llmwiki_rag import _source_from_assistant_result, _source_from_result
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
@@ -194,3 +195,264 @@ async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reimport_retires_pages_removed_from_source_snapshot(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
common = {
|
||||
"source_type": "weknora",
|
||||
"source_key": "kb-refresh",
|
||||
"source_name": "自动同步普通库",
|
||||
"trigger": "vectorization_completed",
|
||||
"created_by": "admin",
|
||||
"mapping_id": "kb-refresh",
|
||||
}
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "保留内容"},
|
||||
{"slug": "wiki/removed", "title": "删除页", "content": "旧内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "新内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"], include_chunks=True)
|
||||
assert [page["slug"] for page in pages] == ["wiki/kept"]
|
||||
assert pages[0]["content_markdown"] == "新内容"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_can_be_renamed_and_deleted_without_affecting_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="待编辑助手库",
|
||||
description="旧描述",
|
||||
)
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/delete-test", "title": "删除测试", "content": "保留在总库中的内容"}
|
||||
],
|
||||
"entities": [{"local_id": "e1", "type": "concept", "name": "删除测试实体"}],
|
||||
"relations": [],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-delete-test",
|
||||
source_name="删除测试普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-delete-test",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
updated = await repository.update_base(
|
||||
custom_base["id"],
|
||||
name="已编辑助手库",
|
||||
description="新描述",
|
||||
)
|
||||
assert updated is not None
|
||||
assert updated["name"] == "已编辑助手库"
|
||||
assert updated["description"] == "新描述"
|
||||
|
||||
assert await repository.delete_base(custom_base["id"])
|
||||
assert await repository.get_base(custom_base["id"]) is None
|
||||
assert all(base["id"] != custom_base["id"] for base in await repository.list_bases())
|
||||
assert await repository.list_pages(base_id=global_base["id"], query="保留在总库")
|
||||
with pytest.raises(ValueError, match="不能删除"):
|
||||
await repository.delete_base(global_base["id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_base_aligns_same_entity_title_and_searches_knowledge_body(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
first_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-one", "knowledge_base_name": "人员库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "people/zhang-san",
|
||||
"title": "张 三",
|
||||
"page_type": "article",
|
||||
"content": "张三负责设备巡检。",
|
||||
"summary": "摘要专用词:月度综述。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "one",
|
||||
"type": "人物",
|
||||
"name": "张 三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "people/zhang-san",
|
||||
}
|
||||
],
|
||||
}
|
||||
second_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-two", "knowledge_base_name": "职责库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "entity/person-001",
|
||||
"title": "Zhang San",
|
||||
"page_type": "concept",
|
||||
"content": "张三负责系统故障处置。",
|
||||
"summary": "另一条摘要。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "two",
|
||||
"type": "person",
|
||||
"name": "张三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "entity/person-001",
|
||||
}
|
||||
],
|
||||
}
|
||||
for package, source_key in ((first_package, "kb-one"), (second_package, "kb-two")):
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=source_key,
|
||||
source_name=source_key,
|
||||
trigger="vectorization_completed",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert "张三负责设备巡检。" in pages[0]["content_markdown"]
|
||||
assert "张三负责系统故障处置。" in pages[0]["content_markdown"]
|
||||
assert len(pages[0]["origins"]) == 2
|
||||
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="月度综述",
|
||||
embedding_client=None,
|
||||
) == []
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="系统故障处置",
|
||||
embedding_client=None,
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert "系统故障处置" in results[0]["matched_section"]["content"]
|
||||
assert "月度综述" not in results[0]["matched_section"]["content"]
|
||||
|
||||
async with repository._sf() as session:
|
||||
entities = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityRow).where(
|
||||
AssistantKnowledgeEntityRow.base_id == base["id"],
|
||||
AssistantKnowledgeEntityRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(entities) == 1
|
||||
contributions = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityContributionRow).where(
|
||||
AssistantKnowledgeEntityContributionRow.entity_id == entities[0].id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(contributions) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uploaded_package_reuses_embedded_weknora_source_identity(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
for upload_no, content in ((1, "旧知识"), (2, "新知识")):
|
||||
job = await repository.create_queued_import_job(
|
||||
source_type="weknora",
|
||||
source_key=f"upload:{upload_no}",
|
||||
source_name=f"upload-{upload_no}.jsonl",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
base_id=base["id"],
|
||||
)
|
||||
await repository.import_package_for_job(
|
||||
job["id"],
|
||||
package={
|
||||
"source": {
|
||||
"provider": "weknora",
|
||||
"knowledge_base_id": "stable-kb-id",
|
||||
"knowledge_base_name": "同一个普通知识库",
|
||||
},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "wiki/stable",
|
||||
"title": "稳定实体",
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
},
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
active_sources = await repository.list_sources(base_id=base["id"])
|
||||
assert len(active_sources) == 1
|
||||
assert active_sources[0]["source_key"] == "stable-kb-id"
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert pages[0]["content_markdown"] == "新知识"
|
||||
|
||||
|
||||
def test_assistant_citation_uses_matched_knowledge_instead_of_summary() -> None:
|
||||
source = _source_from_assistant_result(
|
||||
{
|
||||
"id": "page-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content_markdown": "完整页面正文",
|
||||
"matched_section": {
|
||||
"content": "实际命中的知识片段:设备过热时先断电。",
|
||||
},
|
||||
"score": 0.91,
|
||||
},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert source["snippet"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
|
||||
|
||||
def test_regular_wiki_citation_does_not_fall_back_to_summary() -> None:
|
||||
source = _source_from_result(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"wiki_page_id": "wiki-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content": "实际知识正文",
|
||||
"matched_sections": [],
|
||||
},
|
||||
{"id": "kb-1", "name": "普通知识库"},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际知识正文"
|
||||
assert source["snippet"] == "实际知识正文"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
@@ -199,7 +199,7 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
async def test_auto_sync_scheduler_scans_immediately_then_waits(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
@@ -208,6 +208,8 @@ async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
|
||||
async def fake_sync_once(app) -> None:
|
||||
calls.append("sync")
|
||||
if calls.count("sync") > 1:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
@@ -224,7 +226,14 @@ async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert calls == ["sleep:120"]
|
||||
assert calls == ["sync", "sleep:120"]
|
||||
|
||||
|
||||
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
|
||||
config = LocalWikiIndexConfig()
|
||||
|
||||
assert config.auto_sync is True
|
||||
assert config.sync_interval_seconds == 30
|
||||
|
||||
|
||||
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, UploadFile
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.routers import wiki_packages
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import write_package
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index import embedding as embedding_module
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
@@ -146,7 +149,7 @@ async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
|
||||
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch, tmp_path):
|
||||
observed = {}
|
||||
|
||||
class LocalModel:
|
||||
@@ -162,13 +165,115 @@ async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch):
|
||||
return [[1.0, *([0.0] * 1023)] for _ in texts]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
|
||||
config = LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024, batch_size=2, local_threads=2)
|
||||
(tmp_path / "onnx").mkdir()
|
||||
(tmp_path / "onnx" / "model.onnx").touch()
|
||||
config = LocalWikiEmbeddingConfig(
|
||||
provider="local",
|
||||
model="bge-embedding-m3",
|
||||
dimensions=1024,
|
||||
batch_size=2,
|
||||
local_threads=2,
|
||||
local_model_path=str(tmp_path),
|
||||
local_process_isolation=False,
|
||||
)
|
||||
client = StrictWikiEmbeddingClient(config)
|
||||
assert len(await client.embed_query("本地查询")) == 1024
|
||||
assert observed["model_name"] == "BAAI/bge-m3"
|
||||
assert observed["providers"] == ["CPUExecutionProvider"]
|
||||
assert observed["local_files_only"] is True
|
||||
assert observed["specific_model_path"] == str(tmp_path)
|
||||
assert observed["batch_size"] == observed["threads"] == 2
|
||||
assert client._client is None
|
||||
assert client.local_encoder_status()["state"] == "ready"
|
||||
assert client.local_encoder_status()["model_files_ready"] is True
|
||||
|
||||
cold_client = StrictWikiEmbeddingClient(config)
|
||||
status = cold_client.start_local_encoder()
|
||||
assert status["state"] == "starting"
|
||||
assert status["running"] is True
|
||||
await cold_client._local_warmup_task
|
||||
assert cold_client.local_encoder_status()["state"] == "ready"
|
||||
|
||||
monkeypatch.delenv("DEERFLOW_BGE_M3_MODEL_PATH", raising=False)
|
||||
missing_path = StrictWikiEmbeddingClient(
|
||||
LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024),
|
||||
)
|
||||
missing_path.start_local_encoder()
|
||||
await missing_path._local_warmup_task
|
||||
assert missing_path.local_encoder_status()["state"] == "failed"
|
||||
assert "目录未配置" in str(missing_path.local_encoder_status()["last_error"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_uses_offline_isolated_worker(monkeypatch, tmp_path):
|
||||
(tmp_path / "onnx").mkdir()
|
||||
(tmp_path / "onnx" / "model.onnx").touch()
|
||||
observed: dict = {}
|
||||
|
||||
class FakeStdout:
|
||||
def __init__(self):
|
||||
self.lines = [json.dumps({"status": "ready"}) + "\n"]
|
||||
|
||||
def readline(self):
|
||||
return self.lines.pop(0) if self.lines else ""
|
||||
|
||||
class FakeProcess:
|
||||
def __init__(self):
|
||||
self.stdout = FakeStdout()
|
||||
self.returncode = None
|
||||
self.stdin = SimpleNamespace(write=self.write, flush=lambda: None)
|
||||
|
||||
def write(self, line):
|
||||
payload = json.loads(line)
|
||||
if payload.get("command") == "shutdown":
|
||||
self.returncode = 0
|
||||
return
|
||||
self.stdout.lines.append(
|
||||
json.dumps(
|
||||
{
|
||||
"id": payload["id"],
|
||||
"status": "ok",
|
||||
"vectors": [[1.0, *([0.0] * 1023)] for _ in payload["texts"]],
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return self.returncode
|
||||
|
||||
def terminate(self):
|
||||
self.returncode = -15
|
||||
|
||||
def kill(self):
|
||||
self.returncode = -9
|
||||
|
||||
process = FakeProcess()
|
||||
|
||||
def fake_popen(command, **kwargs):
|
||||
observed.update({"command": command, **kwargs})
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(embedding_module.subprocess, "Popen", fake_popen)
|
||||
client = StrictWikiEmbeddingClient(
|
||||
LocalWikiEmbeddingConfig(
|
||||
provider="local",
|
||||
model="bge-embedding-m3",
|
||||
dimensions=1024,
|
||||
local_model_path=str(tmp_path),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(await client.embed_query("隔离编码")) == 1024
|
||||
assert "deerflow.integrations.weknora.local_index.embedding_worker" in observed["command"]
|
||||
assert observed["env"]["HF_HUB_OFFLINE"] == "1"
|
||||
assert observed["env"]["PYTHONIOENCODING"] == "utf-8"
|
||||
assert client.local_encoder_status()["process_isolation"] is True
|
||||
await client.close()
|
||||
assert process.poll() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -178,7 +283,7 @@ async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypat
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def admin(_):
|
||||
return "admin"
|
||||
return "admin", True
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, *args, **kwargs):
|
||||
@@ -193,11 +298,61 @@ async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypat
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
# Deliberately no skill_store ownership row: built-ins live on disk.
|
||||
monkeypatch.setattr(skill_knowledge, "_require_admin", admin)
|
||||
monkeypatch.setattr(skill_knowledge, "_actor", admin)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
assert job["skill_names"] == ["builtin"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_regular_user_can_distill_visible_skill_only_to_writable_wiki(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def regular_user(_):
|
||||
return "user-1", False
|
||||
|
||||
class Skills:
|
||||
async def get_visible(self, name, user_id):
|
||||
assert user_id == "user-1"
|
||||
return {"name": name} if name == "visible-skill" else None
|
||||
|
||||
async def list_favorite_skill_names(self, user_id):
|
||||
assert user_id == "user-1"
|
||||
return []
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, mapping_id, user_id, *, write, is_admin):
|
||||
assert (mapping_id, user_id, write, is_admin) == ("mapping", "user-1", True, False)
|
||||
return {"name": "我的 Wiki", "weknora_id": "remote"}
|
||||
|
||||
class Jobs:
|
||||
async def create_job(self, **kwargs):
|
||||
return kwargs
|
||||
|
||||
app = FastAPI()
|
||||
app.state.config = None
|
||||
app.state.skill_store = Skills()
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
monkeypatch.setattr(skill_knowledge, "_actor", regular_user)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
monkeypatch.setattr(
|
||||
skill_knowledge,
|
||||
"get_or_new_skill_storage",
|
||||
lambda **_: SimpleNamespace(get_skills_root_path=lambda: tmp_path),
|
||||
)
|
||||
body = skill_knowledge.CreateSyncJobRequest(
|
||||
skill_names=["visible-skill"],
|
||||
targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}],
|
||||
review_mode="required",
|
||||
)
|
||||
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
|
||||
assert job["created_by"] == "user-1"
|
||||
assert job["review_mode"] == "off"
|
||||
|
||||
|
||||
@@ -205,3 +360,70 @@ def test_short_wiki_summary_and_body_still_have_a_vector_section():
|
||||
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
|
||||
assert len(sections) == 1
|
||||
assert "降低设备温度" in sections[0].embedding_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_can_upload_and_activate_offline_encoder(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import llmwiki_index
|
||||
from deerflow.config.system_settings import SystemSettings
|
||||
|
||||
async def admin(_):
|
||||
return "admin", True
|
||||
|
||||
class Embedding:
|
||||
def __init__(self):
|
||||
self.model_path = ""
|
||||
|
||||
async def reset_local_encoder(self, *, model_path=None):
|
||||
if model_path is not None:
|
||||
self.model_path = model_path
|
||||
|
||||
def local_encoder_status(self):
|
||||
return {
|
||||
"provider": "local",
|
||||
"model": "bge-embedding-m3",
|
||||
"state": "idle",
|
||||
"model_path": self.model_path,
|
||||
"model_files_ready": bool(self.model_path),
|
||||
}
|
||||
|
||||
saved = []
|
||||
monkeypatch.setattr(llmwiki_index, "_actor", admin)
|
||||
monkeypatch.setattr(llmwiki_index, "runtime_home", lambda: tmp_path)
|
||||
monkeypatch.setattr(llmwiki_index, "load_system_settings", SystemSettings)
|
||||
monkeypatch.setattr(llmwiki_index, "save_system_settings", lambda value: saved.append(value) or value)
|
||||
embedding = Embedding()
|
||||
app = FastAPI()
|
||||
app.state.config = SimpleNamespace(
|
||||
llmwiki=SimpleNamespace(
|
||||
local_wiki_index=SimpleNamespace(embedding=SimpleNamespace(provider="local"))
|
||||
)
|
||||
)
|
||||
app.state.llmwiki_embedding = embedding
|
||||
request = Request({"type": "http", "headers": [], "app": app})
|
||||
files = [
|
||||
UploadFile(filename="model.onnx", file=BytesIO(b"onnx")),
|
||||
UploadFile(filename="config.json", file=BytesIO(b"{}")),
|
||||
]
|
||||
|
||||
result = await llmwiki_index.upload_local_encoder(
|
||||
request,
|
||||
files=files,
|
||||
paths=["onnx/model.onnx", "config.json"],
|
||||
)
|
||||
|
||||
active = tmp_path / "models" / "bge-m3"
|
||||
assert (active / "onnx" / "model.onnx").read_bytes() == b"onnx"
|
||||
assert result["uploaded_files"] == 2
|
||||
assert saved[-1].local_wiki_encoder.model_path == str(active.resolve())
|
||||
assert embedding.model_path == str(active.resolve())
|
||||
|
||||
|
||||
def test_encoder_upload_rejects_parent_traversal():
|
||||
from app.gateway.routers import llmwiki_index
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
llmwiki_index._safe_relative_upload_path("../model.onnx")
|
||||
assert getattr(exc_info.value, "status_code", None) == 422
|
||||
Reference in new issue
Block a user