Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-code into master-nj
Co-authored-by: Cursor <cursoragent@cursor.com> # Conflicts: # frontend-web/src/components/workspace/workspace-nav-chat-list.tsx # frontend-web/src/pages/WorkspaceRoutes.tsx # offline-backend-20260512/backend/app/gateway/app.py # offline-backend-20260512/backend/app/gateway/deps.py # offline-backend-20260512/backend/pyproject.toml
This commit is contained in:
commit
cf9fe58905
109 files changed
+18863
-982
No files matched your search
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway import deps
|
||||
from app.gateway.routers import artifacts
|
||||
|
||||
|
||||
class _FakeThreadStore:
|
||||
def __init__(self, rows: list[dict]) -> None:
|
||||
self.rows = rows
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def search(self, **kwargs) -> list[dict]:
|
||||
self.calls.append(kwargs)
|
||||
offset = int(kwargs.get("offset", 0))
|
||||
limit = int(kwargs.get("limit", 100))
|
||||
return self.rows[offset : offset + limit]
|
||||
|
||||
|
||||
class _FakeAgentStore:
|
||||
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict]:
|
||||
assert user_id == "owner"
|
||||
agents = {
|
||||
"analysis-agent": {
|
||||
"id": "analysis-agent",
|
||||
"name": "舆情分析智能体",
|
||||
}
|
||||
}
|
||||
return [agents[agent_id] for agent_id in agent_ids if agent_id in agents]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
|
||||
thread_one = tmp_path / "thread-one" / "outputs"
|
||||
thread_two = tmp_path / "thread-two" / "outputs"
|
||||
thread_one.mkdir(parents=True)
|
||||
thread_two.mkdir(parents=True)
|
||||
|
||||
report = thread_one / "研判报告.docx"
|
||||
table = thread_one / "数据汇总.xlsx"
|
||||
image = thread_one / "配图.png"
|
||||
audio = thread_two / "录音.mp3"
|
||||
notes = thread_two / "结论.md"
|
||||
archive = thread_two / "证据.zip"
|
||||
for path, content in (
|
||||
(report, b"doc"),
|
||||
(table, b"sheet"),
|
||||
(image, b"image"),
|
||||
(audio, b"audio"),
|
||||
(notes, b"# notes"),
|
||||
(archive, b"zip"),
|
||||
):
|
||||
path.write_bytes(content)
|
||||
|
||||
os.utime(report, (100, 100))
|
||||
os.utime(table, (200, 200))
|
||||
os.utime(notes, (300, 300))
|
||||
os.utime(archive, (400, 400))
|
||||
|
||||
rows = [
|
||||
{
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "战场环境综合研判",
|
||||
"metadata": {"agent_id": "analysis-agent"},
|
||||
},
|
||||
{
|
||||
"thread_id": "thread-two",
|
||||
"display_name": "生成发展报告",
|
||||
"metadata": {},
|
||||
},
|
||||
]
|
||||
store = _FakeThreadStore(rows)
|
||||
|
||||
class _Paths:
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
|
||||
assert user_id == "owner"
|
||||
return tmp_path / thread_id / "outputs"
|
||||
|
||||
async def _resolve_owner(_thread_id: str) -> str:
|
||||
return "owner"
|
||||
|
||||
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
|
||||
items = await artifacts._scan_artifact_library(store)
|
||||
|
||||
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
|
||||
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
|
||||
assert [item["source_type"] for item in items] == ["general_qa", "general_qa", "agent", "agent"]
|
||||
assert items[-1]["thread_title"] == "战场环境综合研判"
|
||||
assert items[-1]["agent_id"] == "analysis-agent"
|
||||
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
|
||||
assert store.calls[0]["exclude_system"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
|
||||
output_dir = tmp_path / "thread-one" / "outputs"
|
||||
agent_output_dir = tmp_path / "thread-agent" / "outputs"
|
||||
output_dir.mkdir(parents=True)
|
||||
agent_output_dir.mkdir(parents=True)
|
||||
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
|
||||
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
|
||||
(agent_output_dir / "智能体结论.md").write_text("agent report", encoding="utf-8")
|
||||
|
||||
store = _FakeThreadStore(
|
||||
[
|
||||
{
|
||||
"thread_id": "thread-one",
|
||||
"display_name": "任务研判对话",
|
||||
"metadata": {},
|
||||
},
|
||||
{
|
||||
"thread_id": "thread-agent",
|
||||
"display_name": "智能体研判对话",
|
||||
"metadata": {"agent_id": "analysis-agent"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
class _Paths:
|
||||
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
|
||||
return tmp_path / thread_id / "outputs"
|
||||
|
||||
async def _resolve_owner(_thread_id: str) -> str:
|
||||
return "owner"
|
||||
|
||||
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
|
||||
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
|
||||
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
|
||||
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
|
||||
monkeypatch.setattr(deps, "get_agent_store", lambda _request: _FakeAgentStore())
|
||||
artifacts._artifact_library_cache.clear()
|
||||
|
||||
raw_endpoint = artifacts.list_artifact_library.__wrapped__
|
||||
response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query="研判对话",
|
||||
file_kind="text_data",
|
||||
source_type="general_qa",
|
||||
offset=0,
|
||||
limit=1,
|
||||
refresh=True,
|
||||
)
|
||||
|
||||
assert response.total == 1
|
||||
assert response.offset == 0
|
||||
assert response.limit == 1
|
||||
assert [item.name for item in response.items] == ["任务报告.md"]
|
||||
|
||||
spreadsheet_response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query="任务数据",
|
||||
file_kind="spreadsheet",
|
||||
source_type="all",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert spreadsheet_response.total == 1
|
||||
assert spreadsheet_response.items[0].name == "任务数据.csv"
|
||||
|
||||
agent_response = await raw_endpoint(
|
||||
request=SimpleNamespace(),
|
||||
query=None,
|
||||
file_kind="all",
|
||||
source_type="agent",
|
||||
offset=0,
|
||||
limit=50,
|
||||
refresh=False,
|
||||
)
|
||||
assert agent_response.total == 1
|
||||
assert agent_response.items[0].source_type == "agent"
|
||||
assert agent_response.items[0].agent_name == "舆情分析智能体"
|
||||
assert agent_response.items[0].name == "智能体结论.md"
|
||||
@@ -0,0 +1,458 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.llmwiki_rag import _source_from_assistant_result, _source_from_result
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
AssistantKnowledgeEntityRow,
|
||||
)
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class FakeEmbeddingClient:
|
||||
dimensions = 3
|
||||
fingerprint = "test-bge-m3"
|
||||
|
||||
async def embed_query(self, _query: str) -> list[float]:
|
||||
return [1.0, 0.0, 0.0]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
yield AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_base_entity_alignment_search_and_revision(repository: AssistantKnowledgeRepository) -> None:
|
||||
first = await repository.initialize(created_by="admin")
|
||||
second = await repository.initialize(created_by="someone-else")
|
||||
assert first["id"] == second["id"]
|
||||
|
||||
package = {
|
||||
"source": {"type": "skill"},
|
||||
"wiki_pages": [{"slug": "skills/demo/overview", "title": "示例知识", "content": "张三负责测试。"}],
|
||||
"entities": [{"local_id": "e1", "type": "person", "name": "张三", "aliases": ["Zhang San"]}],
|
||||
"relations": [],
|
||||
"documents": [],
|
||||
"chunks": [],
|
||||
"graph": {"nodes": [], "edges": []},
|
||||
}
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key="demo",
|
||||
source_name="示例技能",
|
||||
trigger="skill_direct",
|
||||
created_by="admin",
|
||||
skill_name="demo",
|
||||
)
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key="kb-1",
|
||||
source_name="普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
results = await repository.list_pages(query="张三")
|
||||
alias_results = await repository.list_pages(query="Zhang San")
|
||||
revisions = await repository.list_revisions("skills/demo/overview")
|
||||
assert {row["page_type"] for row in results} == {"source", "entity"}
|
||||
assert [row["page_type"] for row in alias_results] == ["entity"]
|
||||
assert len(revisions) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
package = {
|
||||
"source": {},
|
||||
"wiki_pages": [],
|
||||
"entities": [{"local_id": "e1", "type": "concept", "name": "共享实体"}],
|
||||
"relations": [],
|
||||
}
|
||||
for source_key in ("one", "two"):
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key=source_key,
|
||||
source_name=source_key,
|
||||
trigger="skill_direct",
|
||||
created_by="admin",
|
||||
skill_name=source_key,
|
||||
)
|
||||
|
||||
sources = await repository.list_sources()
|
||||
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "one"))
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow))).scalars().all()
|
||||
assert entity.status == "active"
|
||||
assert len(contributions) == 1
|
||||
|
||||
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "two"))
|
||||
async with repository._sf() as session:
|
||||
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
|
||||
assert entity.status == "deleted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_import_is_also_mirrored_to_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="专题助手库",
|
||||
description="只承接某个专题的 WeKnora 导出数据",
|
||||
)
|
||||
package = {
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/topic", "title": "专题 Wiki", "content": "王五负责统筹。"}],
|
||||
"entities": [{"local_id": "e1", "type": "person", "name": "王五"}],
|
||||
"relations": [],
|
||||
}
|
||||
|
||||
job = await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key="kb-topic",
|
||||
source_name="专题普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-topic",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
assert job["base_id"] == custom_base["id"]
|
||||
assert job["mirrored_global_base_id"] == global_base["id"]
|
||||
assert len(await repository.list_pages(base_id=custom_base["id"], query="王五")) == 2
|
||||
assert len(await repository.list_pages(base_id=global_base["id"], query="王五")) == 2
|
||||
bases = await repository.list_bases()
|
||||
assert [base["id"] for base in bases[:2]] == [global_base["id"], custom_base["id"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
job = await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [{"slug": "wiki/vector", "title": "向量页", "content": "页面正文"}],
|
||||
"vectors": [
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 0,
|
||||
"section_content": "张三负责向量化检索。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_fingerprint": "test-bge-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [1.0, 0.0, 0.0],
|
||||
},
|
||||
{
|
||||
"wiki_slug": "wiki/vector",
|
||||
"section_index": 1,
|
||||
"section_content": "无关片段。",
|
||||
"embedding_model": "bge-embedding-m3",
|
||||
"embedding_fingerprint": "test-bge-m3",
|
||||
"embedding_dimensions": 3,
|
||||
"vector": [0.0, 1.0, 0.0],
|
||||
},
|
||||
],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-vector",
|
||||
source_name="向量普通库",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
assert job["counts"]["vectors"] == 2
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
)
|
||||
assert results[0]["retrieval_mode"] == "vector"
|
||||
assert results[0]["matched_section"]["embedding_model"] == "bge-embedding-m3"
|
||||
assert results[0]["content_markdown"] == "张三负责向量化检索。"
|
||||
|
||||
sources = await repository.list_sources()
|
||||
assert await repository.delete_source(sources[0]["id"])
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reimport_retires_pages_removed_from_source_snapshot(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
common = {
|
||||
"source_type": "weknora",
|
||||
"source_key": "kb-refresh",
|
||||
"source_name": "自动同步普通库",
|
||||
"trigger": "vectorization_completed",
|
||||
"created_by": "admin",
|
||||
"mapping_id": "kb-refresh",
|
||||
}
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "保留内容"},
|
||||
{"slug": "wiki/removed", "title": "删除页", "content": "旧内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "新内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"], include_chunks=True)
|
||||
assert [page["slug"] for page in pages] == ["wiki/kept"]
|
||||
assert pages[0]["content_markdown"] == "新内容"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_can_be_renamed_and_deleted_without_affecting_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="待编辑助手库",
|
||||
description="旧描述",
|
||||
)
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/delete-test", "title": "删除测试", "content": "保留在总库中的内容"}
|
||||
],
|
||||
"entities": [{"local_id": "e1", "type": "concept", "name": "删除测试实体"}],
|
||||
"relations": [],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-delete-test",
|
||||
source_name="删除测试普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-delete-test",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
updated = await repository.update_base(
|
||||
custom_base["id"],
|
||||
name="已编辑助手库",
|
||||
description="新描述",
|
||||
)
|
||||
assert updated is not None
|
||||
assert updated["name"] == "已编辑助手库"
|
||||
assert updated["description"] == "新描述"
|
||||
|
||||
assert await repository.delete_base(custom_base["id"])
|
||||
assert await repository.get_base(custom_base["id"]) is None
|
||||
assert all(base["id"] != custom_base["id"] for base in await repository.list_bases())
|
||||
assert await repository.list_pages(base_id=global_base["id"], query="保留在总库")
|
||||
with pytest.raises(ValueError, match="不能删除"):
|
||||
await repository.delete_base(global_base["id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_base_aligns_same_entity_title_and_searches_knowledge_body(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
first_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-one", "knowledge_base_name": "人员库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "people/zhang-san",
|
||||
"title": "张 三",
|
||||
"page_type": "article",
|
||||
"content": "张三负责设备巡检。",
|
||||
"summary": "摘要专用词:月度综述。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "one",
|
||||
"type": "人物",
|
||||
"name": "张 三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "people/zhang-san",
|
||||
}
|
||||
],
|
||||
}
|
||||
second_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-two", "knowledge_base_name": "职责库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "entity/person-001",
|
||||
"title": "Zhang San",
|
||||
"page_type": "concept",
|
||||
"content": "张三负责系统故障处置。",
|
||||
"summary": "另一条摘要。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "two",
|
||||
"type": "person",
|
||||
"name": "张三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "entity/person-001",
|
||||
}
|
||||
],
|
||||
}
|
||||
for package, source_key in ((first_package, "kb-one"), (second_package, "kb-two")):
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=source_key,
|
||||
source_name=source_key,
|
||||
trigger="vectorization_completed",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert "张三负责设备巡检。" in pages[0]["content_markdown"]
|
||||
assert "张三负责系统故障处置。" in pages[0]["content_markdown"]
|
||||
assert len(pages[0]["origins"]) == 2
|
||||
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="月度综述",
|
||||
embedding_client=None,
|
||||
) == []
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="系统故障处置",
|
||||
embedding_client=None,
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert "系统故障处置" in results[0]["matched_section"]["content"]
|
||||
assert "月度综述" not in results[0]["matched_section"]["content"]
|
||||
|
||||
async with repository._sf() as session:
|
||||
entities = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityRow).where(
|
||||
AssistantKnowledgeEntityRow.base_id == base["id"],
|
||||
AssistantKnowledgeEntityRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(entities) == 1
|
||||
contributions = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityContributionRow).where(
|
||||
AssistantKnowledgeEntityContributionRow.entity_id == entities[0].id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(contributions) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uploaded_package_reuses_embedded_weknora_source_identity(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
for upload_no, content in ((1, "旧知识"), (2, "新知识")):
|
||||
job = await repository.create_queued_import_job(
|
||||
source_type="weknora",
|
||||
source_key=f"upload:{upload_no}",
|
||||
source_name=f"upload-{upload_no}.jsonl",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
base_id=base["id"],
|
||||
)
|
||||
await repository.import_package_for_job(
|
||||
job["id"],
|
||||
package={
|
||||
"source": {
|
||||
"provider": "weknora",
|
||||
"knowledge_base_id": "stable-kb-id",
|
||||
"knowledge_base_name": "同一个普通知识库",
|
||||
},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "wiki/stable",
|
||||
"title": "稳定实体",
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
},
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
active_sources = await repository.list_sources(base_id=base["id"])
|
||||
assert len(active_sources) == 1
|
||||
assert active_sources[0]["source_key"] == "stable-kb-id"
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert pages[0]["content_markdown"] == "新知识"
|
||||
|
||||
|
||||
def test_assistant_citation_uses_matched_knowledge_instead_of_summary() -> None:
|
||||
source = _source_from_assistant_result(
|
||||
{
|
||||
"id": "page-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content_markdown": "完整页面正文",
|
||||
"matched_section": {
|
||||
"content": "实际命中的知识片段:设备过热时先断电。",
|
||||
},
|
||||
"score": 0.91,
|
||||
},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert source["snippet"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
|
||||
|
||||
def test_regular_wiki_citation_does_not_fall_back_to_summary() -> None:
|
||||
source = _source_from_result(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"wiki_page_id": "wiki-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content": "实际知识正文",
|
||||
"matched_sections": [],
|
||||
},
|
||||
{"id": "kb-1", "name": "普通知识库"},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际知识正文"
|
||||
assert source["snippet"] == "实际知识正文"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Regression coverage for the openGauss/GaussDB persistence backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.schema import CreateIndex, CreateTable
|
||||
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
(
|
||||
"opengauss://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
(
|
||||
"postgresql://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow?ssl=require",
|
||||
),
|
||||
(
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_gauss_urls_always_select_the_opengauss_async_dialect(
|
||||
configured: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
config = DatabaseConfig(backend="gauss", gauss_url=configured)
|
||||
assert config.app_sqlalchemy_url == expected
|
||||
|
||||
|
||||
def test_gauss_backend_requires_a_supported_nonempty_url() -> None:
|
||||
with pytest.raises(ValueError, match="gauss_url is required"):
|
||||
_ = DatabaseConfig(backend="gauss").app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValueError, match="must use opengauss"):
|
||||
_ = DatabaseConfig(
|
||||
backend="gauss",
|
||||
gauss_url="mysql://user:pass@127.0.0.1/deerflow",
|
||||
).app_sqlalchemy_url
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
DatabaseConfig(backend="unknown")
|
||||
|
||||
|
||||
def test_gauss_timestamps_keep_timezone_information() -> None:
|
||||
dialect = type("GaussDialect", (), {"name": "opengauss"})()
|
||||
value = datetime(2026, 9, 2, 8, 30, tzinfo=UTC)
|
||||
column_type = BeijingDateTime()
|
||||
|
||||
assert column_type.process_bind_param(value, dialect) is value
|
||||
assert column_type.process_result_value(value, dialect) is value
|
||||
|
||||
|
||||
def test_official_async_dialect_compiles_the_complete_orm_schema() -> None:
|
||||
"""Every registered table/index must compile before startup touches GaussDB."""
|
||||
pytest.importorskip("opengauss_sqlalchemy")
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 -- register ORM metadata
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
dialect_cls = make_url("opengauss+asyncpg://user:pass@localhost/db").get_dialect(
|
||||
_is_async=True
|
||||
)
|
||||
dialect = dialect_cls()
|
||||
|
||||
assert dialect.__class__.__module__.startswith("opengauss_sqlalchemy")
|
||||
assert Base.metadata.sorted_tables
|
||||
for table in Base.metadata.sorted_tables:
|
||||
assert str(CreateTable(table).compile(dialect=dialect)).startswith("\nCREATE TABLE")
|
||||
for index in table.indexes:
|
||||
assert str(CreateIndex(index).compile(dialect=dialect)).startswith("CREATE")
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Ordinary knowledge-base editing and remote-deletion reconciliation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import FastAPI
|
||||
|
||||
from deerflow.integrations.weknora.client import WeKnoraClient
|
||||
from deerflow.persistence.llmwiki import MemoryLlmWikiStore
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def knowledge_api(monkeypatch: pytest.MonkeyPatch):
|
||||
from app.gateway.routers import llmwiki
|
||||
|
||||
store = MemoryLlmWikiStore()
|
||||
row = await store.create_mapping(weknora_id="remote-one", owner_user_id="owner", name="原名称", description="原描述", kb_type="document")
|
||||
await store.request_publish(row["id"], "owner", is_admin=False)
|
||||
state = SimpleNamespace(
|
||||
store=store,
|
||||
mapping_id=row["id"],
|
||||
remote={"remote-one": {"id": "remote-one", "name": "原名称", "description": "原描述", "knowledge_count": 3}},
|
||||
user_id="owner",
|
||||
is_admin=False,
|
||||
upstream_status=200,
|
||||
writes=[],
|
||||
)
|
||||
|
||||
async def upstream(request: httpx.Request) -> httpx.Response:
|
||||
if state.upstream_status != 200:
|
||||
return httpx.Response(state.upstream_status, json={"message": "知识服务暂不可用"})
|
||||
if request.url.path == "/api/v1/knowledge-bases":
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(200, json={"data": list(state.remote.values())})
|
||||
remote_id = request.url.path.rsplit("/", 1)[-1]
|
||||
if remote_id not in state.remote:
|
||||
return httpx.Response(404)
|
||||
if request.method == "PUT":
|
||||
changes = json.loads(request.content)
|
||||
state.writes.append(changes)
|
||||
state.remote[remote_id].update(changes)
|
||||
else:
|
||||
assert request.method == "GET"
|
||||
return httpx.Response(200, json={"data": state.remote[remote_id]})
|
||||
|
||||
async def actor(_: object) -> tuple[str, bool]:
|
||||
return state.user_id, state.is_admin
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(llmwiki.router)
|
||||
monkeypatch.setattr(llmwiki, "_actor", actor)
|
||||
monkeypatch.setattr(llmwiki, "_store", lambda _: store)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream)) as remote_client:
|
||||
client = WeKnoraClient(base_url="http://weknora.test", http_client=remote_client)
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: client)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as api:
|
||||
state.api = api
|
||||
yield state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scope", ["personal", "public", "all"])
|
||||
@pytest.mark.parametrize("is_admin", [False, True])
|
||||
async def test_lists_hide_remote_deleted_bases_without_deleting_mappings(knowledge_api, scope: str, is_admin: bool) -> None:
|
||||
state = knowledge_api
|
||||
state.is_admin = is_admin
|
||||
live = await state.store.create_mapping(weknora_id="remote-live", owner_user_id="owner", name="保留知识库", description="", kb_type="document")
|
||||
await state.store.request_publish(live["id"], "owner", is_admin=False)
|
||||
state.remote["remote-live"] = {"id": "remote-live", "name": "保留知识库"}
|
||||
deleted = state.remote.pop("remote-one")
|
||||
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [live["id"]]
|
||||
assert await state.store.get_by_weknora_id("remote-one") is not None
|
||||
|
||||
state.remote.clear()
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
assert response.json()["knowledge_bases"] == []
|
||||
|
||||
state.remote["remote-one"] = deleted
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": scope})
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [502, 504])
|
||||
async def test_list_upstream_failure_is_not_treated_as_remote_deletion(knowledge_api, status: int) -> None:
|
||||
state = knowledge_api
|
||||
state.upstream_status = status
|
||||
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
|
||||
assert response.status_code >= 500
|
||||
assert await state.store.get_by_weknora_id("remote-one") is not None
|
||||
state.upstream_status = 200
|
||||
response = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
assert [item["id"] for item in response.json()["knowledge_bases"]] == [state.mapping_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "expected_name", "expected_description"),
|
||||
[
|
||||
({"name": " 新名称 ", "description": "新描述"}, "新名称", "新描述"),
|
||||
({"name": "仅修改名称"}, "仅修改名称", "原描述"),
|
||||
({"description": "仅修改描述"}, "原名称", "仅修改描述"),
|
||||
({"name": "清空描述", "description": ""}, "清空描述", ""),
|
||||
],
|
||||
)
|
||||
async def test_edit_synchronizes_remote_and_local_metadata(knowledge_api, changes, expected_name: str, expected_description: str) -> None:
|
||||
state = knowledge_api
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json=changes)
|
||||
|
||||
assert response.status_code == 200
|
||||
item = response.json()
|
||||
assert (item["name"], item["description"]) == (expected_name, expected_description)
|
||||
assert item["remote_status"] == "available"
|
||||
assert item["knowledge_count"] == 3
|
||||
assert item["publication_status"] == "published"
|
||||
local = await state.store.get_by_weknora_id("remote-one")
|
||||
assert (local["name"], local["description"]) == (expected_name, expected_description)
|
||||
assert state.writes == [{**changes, "name": expected_name}]
|
||||
listing = await state.api.get("/api/llmwiki/knowledge-bases", params={"scope": "personal"})
|
||||
assert listing.json()["knowledge_bases"][0]["name"] == expected_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["", " \t ", "x" * 256, "对话沉淀", " 对话沉淀 "])
|
||||
async def test_edit_rejects_invalid_or_reserved_names(knowledge_api, name: str) -> None:
|
||||
state = knowledge_api
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": name})
|
||||
|
||||
assert response.status_code in {409, 422}
|
||||
assert state.writes == []
|
||||
assert (await state.store.get_by_weknora_id("remote-one"))["name"] == "原名称"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("is_admin", [False, True])
|
||||
async def test_only_owner_or_admin_can_edit_a_public_base(knowledge_api, is_admin: bool) -> None:
|
||||
state = knowledge_api
|
||||
state.user_id = "other-user"
|
||||
state.is_admin = is_admin
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
|
||||
|
||||
assert response.status_code == (200 if is_admin else 404)
|
||||
assert len(state.writes) == (1 if is_admin else 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_cannot_edit_system_conversation_deposit(knowledge_api) -> None:
|
||||
state = knowledge_api
|
||||
state.is_admin = True
|
||||
await state.store.adopt_conversation_deposit_mapping(state.mapping_id, name="对话沉淀", description="系统知识库")
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称"})
|
||||
|
||||
assert response.status_code == 403
|
||||
assert state.writes == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [404, 502, 504])
|
||||
async def test_failed_remote_edit_leaves_local_metadata_unchanged(knowledge_api, status: int) -> None:
|
||||
state = knowledge_api
|
||||
state.upstream_status = status
|
||||
response = await state.api.patch(f"/api/llmwiki/knowledge-bases/{state.mapping_id}", json={"name": "新名称", "description": "新描述"})
|
||||
|
||||
if status == 404:
|
||||
assert response.status_code == 404
|
||||
else:
|
||||
assert response.status_code >= 500
|
||||
local = await state.store.get_by_weknora_id("remote-one")
|
||||
assert (local["name"], local["description"]) == ("原名称", "原描述")
|
||||
@@ -66,6 +66,9 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
assert store.write_batch_size == 1
|
||||
assert len(snapshot["rows"]) == 3
|
||||
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
||||
streamed = [row async for row in store.iter_vector_snapshot_rows("mapping-1", "fp", batch_size=2)]
|
||||
assert [row["section_index"] for row in streamed] == [0, 1, 2]
|
||||
assert streamed[0]["page"]["slug"] == "people/alice"
|
||||
page_statuses = await store.list_page_index_status("mapping-1")
|
||||
assert len(page_statuses) == 1
|
||||
assert page_statuses[0] == {
|
||||
|
||||
@@ -199,7 +199,7 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
async def test_auto_sync_scheduler_scans_immediately_then_waits(monkeypatch) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_sleep(seconds: float) -> None:
|
||||
@@ -208,6 +208,8 @@ async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
|
||||
async def fake_sync_once(app) -> None:
|
||||
calls.append("sync")
|
||||
if calls.count("sync") > 1:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||
@@ -224,7 +226,14 @@ async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||
|
||||
assert calls == ["sleep:120"]
|
||||
assert calls == ["sync", "sleep:120"]
|
||||
|
||||
|
||||
def test_local_wiki_index_defaults_to_automatic_sync() -> None:
|
||||
config = LocalWikiIndexConfig()
|
||||
|
||||
assert config.auto_sync is True
|
||||
assert config.sync_interval_seconds == 30
|
||||
|
||||
|
||||
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
|
||||
|
||||
@@ -68,7 +68,7 @@ def _published_row() -> dict[str, object]:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_knowledge_base_list_exposes_only_published_wiki_identity() -> None:
|
||||
async def test_public_knowledge_base_list_exposes_live_safe_metadata_and_generated_index(monkeypatch) -> None:
|
||||
store = PublicListStore(
|
||||
[
|
||||
{
|
||||
@@ -83,21 +83,116 @@ async def test_public_knowledge_base_list_exposes_only_published_wiki_identity()
|
||||
"owner_user_id": "system",
|
||||
"name": "对话沉淀",
|
||||
},
|
||||
{
|
||||
**_published_row(),
|
||||
"id": "stale-kb",
|
||||
"weknora_id": "remote-missing",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
class Client:
|
||||
async def list_knowledge_bases(self):
|
||||
return [
|
||||
{
|
||||
"id": "remote-one",
|
||||
"name": "远端公开知识库",
|
||||
"description": "远端公开说明",
|
||||
"type": "wiki",
|
||||
"knowledge_count": 12,
|
||||
"chunk_count": 34,
|
||||
"processing_count": 1,
|
||||
"share_count": 3,
|
||||
"created_at": "2026-07-01T08:00:00+08:00",
|
||||
"updated_at": "2026-08-01T08:00:00+08:00",
|
||||
"tenant_id": 10001,
|
||||
"creator_id": "secret-creator",
|
||||
"storage_config": {"secret_key": "must-not-leak"},
|
||||
}
|
||||
]
|
||||
|
||||
async def get_knowledge_base(self, knowledge_base_id: str):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
return {
|
||||
"id": knowledge_base_id,
|
||||
"vector_store_name": "System default",
|
||||
"vector_store_source": "env",
|
||||
"vector_store_engine_type": "elasticsearch",
|
||||
"vector_store_status": "available",
|
||||
"wiki_config": {"enabled": True},
|
||||
}
|
||||
|
||||
async def list_wiki_pages(self, knowledge_base_id: str, **kwargs):
|
||||
assert knowledge_base_id == "remote-one"
|
||||
assert kwargs == {"page": 1, "page_size": 500, "query": ""}
|
||||
return {
|
||||
"pages": [
|
||||
{
|
||||
"id": "page-one",
|
||||
"slug": "concept/start",
|
||||
"title": "开始使用",
|
||||
"page_type": "concept",
|
||||
"status": "completed",
|
||||
"created_at": "2026-07-02T08:00:00+08:00",
|
||||
"updated_at": "2026-08-02T08:00:00+08:00",
|
||||
},
|
||||
{
|
||||
"id": "page-two",
|
||||
"slug": "summary/overview",
|
||||
"title": "知识概览",
|
||||
"page_type": "summary",
|
||||
"status": "completed",
|
||||
"created_at": "2026-07-03T08:00:00+08:00",
|
||||
"updated_at": "2026-08-03T08:00:00+08:00",
|
||||
},
|
||||
],
|
||||
"total": 2,
|
||||
"page": 1,
|
||||
"page_size": 500,
|
||||
"total_pages": 1,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||
|
||||
payload = await llmwiki.list_public_knowledge_bases(_request(store))
|
||||
|
||||
assert store.calls == [("__public_wiki__", "public", False)]
|
||||
assert payload == {
|
||||
"knowledge_bases": [
|
||||
{
|
||||
"id": "1063edee-332a-43b4-bbd9-73bf0a708b94",
|
||||
"name": "公开知识库",
|
||||
}
|
||||
],
|
||||
"total": 1,
|
||||
assert payload["total"] == 1
|
||||
item = payload["knowledge_bases"][0]
|
||||
assert item["id"] == "1063edee-332a-43b4-bbd9-73bf0a708b94"
|
||||
assert item["name"] == "远端公开知识库"
|
||||
assert item["description"] == "远端公开说明"
|
||||
assert item["type"] == "wiki"
|
||||
assert item["knowledge_count"] == 12
|
||||
assert item["chunk_count"] == 34
|
||||
assert item["processing_count"] == 1
|
||||
assert item["share_count"] == 3
|
||||
assert item["is_processing"] is True
|
||||
assert item["remote_created_at"] == "2026-07-01T08:00:00+08:00"
|
||||
assert item["remote_updated_at"] == "2026-08-01T08:00:00+08:00"
|
||||
assert item["vector_store_engine_type"] == "elasticsearch"
|
||||
assert item["wiki"] == {
|
||||
"status": "ready",
|
||||
"page_count": 2,
|
||||
"loaded_page_count": 2,
|
||||
"stats_complete": True,
|
||||
"page_type_counts": {"concept": 1, "summary": 1},
|
||||
"page_status_counts": {"completed": 2},
|
||||
"index_mode": "generated",
|
||||
"has_explicit_index": False,
|
||||
"index_slug": "__index__",
|
||||
"default_slug": "__index__",
|
||||
"created_at": "2026-07-02T08:00:00+08:00",
|
||||
"updated_at": "2026-08-03T08:00:00+08:00",
|
||||
"embed_route": "/embed/knowledge/1063edee-332a-43b4-bbd9-73bf0a708b94?wikiSlug=__index__",
|
||||
}
|
||||
assert item["local_index"]["state"] == "not_synced"
|
||||
assert item["local_index"]["fully_vectorized"] is False
|
||||
assert item["owner_user_id"] is None
|
||||
assert "weknora_id" not in item
|
||||
assert "tenant_id" not in item
|
||||
assert "creator_id" not in item
|
||||
assert "storage_config" not in item
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -178,6 +273,35 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
|
||||
assert payload["pages"][0]["aliases"] == []
|
||||
|
||||
|
||||
def test_public_wiki_summary_prefers_an_explicit_index_page() -> None:
|
||||
payload = llmwiki._public_wiki_summary(
|
||||
"mapping-one",
|
||||
[
|
||||
{
|
||||
"slug": "guide/start",
|
||||
"title": "开始使用",
|
||||
"page_type": "article",
|
||||
"status": "completed",
|
||||
},
|
||||
{
|
||||
"slug": "home",
|
||||
"title": "知识首页",
|
||||
"page_type": "index",
|
||||
"status": "completed",
|
||||
},
|
||||
],
|
||||
total=2,
|
||||
complete=True,
|
||||
)
|
||||
|
||||
assert payload["status"] == "ready"
|
||||
assert payload["index_mode"] == "explicit"
|
||||
assert payload["has_explicit_index"] is True
|
||||
assert payload["index_slug"] == "home"
|
||||
assert payload["default_slug"] == "home"
|
||||
assert payload["embed_route"] == "/embed/knowledge/mapping-one?wikiSlug=home"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
|
||||
store = PublicStore(_published_row())
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import pytest
|
||||
|
||||
from deerflow.skill_knowledge.graph_writer import (
|
||||
EDGE_QUERY,
|
||||
NODE_QUERY,
|
||||
execute_plan,
|
||||
parameterized_statements,
|
||||
plan_from_ir,
|
||||
)
|
||||
|
||||
|
||||
class RecordingTransaction:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
async def run(self, query: str, parameters: dict) -> None:
|
||||
self.calls.append((query, parameters))
|
||||
|
||||
|
||||
def _plan():
|
||||
return plan_from_ir(
|
||||
{
|
||||
"entities": [
|
||||
{"local_id": "a", "type": "person", "name": "张三", "confidence": 1},
|
||||
{"local_id": "b", "type": "system", "name": "知识库", "confidence": 1},
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"source_local_id": "a",
|
||||
"target_local_id": "b",
|
||||
"predicate": "uses",
|
||||
"confidence": 0.9,
|
||||
"evidence": [{"file": "SKILL.md", "line": 2}],
|
||||
}
|
||||
],
|
||||
},
|
||||
target_type="weknora",
|
||||
target_id="kb-1",
|
||||
binding_id="binding-1",
|
||||
snapshot_id="snapshot-1",
|
||||
job_id="job-1",
|
||||
)
|
||||
|
||||
|
||||
def test_direct_graph_plan_uses_only_fixed_parameterized_templates() -> None:
|
||||
statements = parameterized_statements(_plan())
|
||||
assert len(statements) == 3
|
||||
assert {row["statement"] for row in statements} == {NODE_QUERY, EDGE_QUERY}
|
||||
assert all("parameters" in row for row in statements)
|
||||
assert all("张三" not in row["statement"] for row in statements)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_graph_writer_obeys_explicit_enable_switch() -> None:
|
||||
transaction = RecordingTransaction()
|
||||
disabled = await execute_plan(transaction, _plan(), enabled=False)
|
||||
assert disabled == {"nodes": 0, "edges": 0}
|
||||
assert transaction.calls == []
|
||||
|
||||
enabled = await execute_plan(transaction, _plan(), enabled=True)
|
||||
assert enabled == {"nodes": 2, "edges": 1}
|
||||
assert len(transaction.calls) == 3
|
||||
@@ -0,0 +1,95 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.skill_knowledge import SkillKnowledgeRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def repository():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
yield SkillKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matrix_job_creates_one_item_and_binding_per_pair(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a", "b", "c"],
|
||||
targets=[
|
||||
{"target_type": "weknora", "target_mode": "wiki", "target_id": "kb-1", "target_name": "一"},
|
||||
{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"},
|
||||
],
|
||||
created_by="admin",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
bindings = await repository.list_bindings()
|
||||
|
||||
assert detail is not None
|
||||
assert detail["total_count"] == 6
|
||||
assert len(detail["items"]) == 6
|
||||
assert len(bindings) == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detach_delete_only_removes_binding_contributions(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a", "b"],
|
||||
targets=[{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"}],
|
||||
created_by="admin",
|
||||
review_mode="off",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
assert detail is not None
|
||||
binding = (await repository.list_bindings(skill_names=["a"]))[0]
|
||||
detached = await repository.detach_binding(binding["id"], delete_contributions=True)
|
||||
assert detached and detached["status"] == "detached_deleted"
|
||||
assert len(await repository.list_bindings()) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_item_claim_is_atomic_and_records_worker_lease(repository: SkillKnowledgeRepository) -> None:
|
||||
job = await repository.create_job(
|
||||
skill_names=["a"],
|
||||
targets=[
|
||||
{
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": "base",
|
||||
"target_name": "知识梳理",
|
||||
}
|
||||
],
|
||||
created_by="admin",
|
||||
)
|
||||
detail = await repository.get_job(job["id"])
|
||||
item_id = detail["items"][0]["id"]
|
||||
|
||||
first = await repository.claim_item(item_id, owner="worker-a")
|
||||
second = await repository.claim_item(item_id, owner="worker-b")
|
||||
|
||||
assert first and first["lease_owner"] == "worker-a"
|
||||
assert first["attempts"] == 1
|
||||
assert second is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_binding_allows_only_one_active_item(repository: SkillKnowledgeRepository) -> None:
|
||||
target = {
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": "base",
|
||||
"target_name": "知识梳理",
|
||||
}
|
||||
first = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
duplicate = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
assert first["total_count"] == 1
|
||||
assert duplicate["total_count"] == 0
|
||||
assert duplicate["status"] == "completed"
|
||||
|
||||
detail = await repository.get_job(first["id"])
|
||||
await repository.update_item(detail["items"][0]["id"], status="failed")
|
||||
retry = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
|
||||
assert retry["total_count"] == 1
|
||||
@@ -0,0 +1,52 @@
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
from deerflow.skill_knowledge import build_skill_ir, scan_skill
|
||||
|
||||
|
||||
def test_scanner_excludes_code_blocks_source_and_sensitive_files(tmp_path: Path) -> None:
|
||||
(tmp_path / "SKILL.md").write_text(
|
||||
"# 示例\n本技能使用 客户画像数据。\n```python\nPASSWORD = 'must-not-enter-ir'\n```\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "worker.py").write_text("# uses hidden system\nSECRET='never-read'", encoding="utf-8")
|
||||
(tmp_path / ".env").write_text("TOKEN=super-secret-value", encoding="utf-8")
|
||||
|
||||
scan = scan_skill("example", tmp_path)
|
||||
ir = build_skill_ir(scan)
|
||||
|
||||
assert scan.skipped_code_count == 1
|
||||
assert scan.blocked_files == [{"path": ".env", "reason": "疑似密钥或敏感配置文件"}]
|
||||
assert "must-not-enter-ir" not in str(ir)
|
||||
assert "never-read" not in str(ir)
|
||||
assert any(relation["predicate"] == "uses" for relation in ir["relations"])
|
||||
|
||||
|
||||
def test_code_only_change_does_not_change_digest(tmp_path: Path) -> None:
|
||||
(tmp_path / "SKILL.md").write_text("# Stable knowledge", encoding="utf-8")
|
||||
code = tmp_path / "worker.py"
|
||||
code.write_text("print(1)", encoding="utf-8")
|
||||
before = scan_skill("example", tmp_path).source_digest
|
||||
code.write_text("print(2)", encoding="utf-8")
|
||||
after = scan_skill("example", tmp_path).source_digest
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_zip_is_inspected_and_only_safe_knowledge_members_are_read(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "materials.zip"
|
||||
with ZipFile(archive, "w") as output:
|
||||
output.writestr("guide.md", "# 操作手册\n仅提炼这段知识。")
|
||||
output.writestr("worker.py", "SECRET = 'code-must-not-be-read'")
|
||||
output.writestr("../escape.md", "path traversal")
|
||||
|
||||
scan = scan_skill("example", tmp_path)
|
||||
|
||||
assert [file.path for file in scan.files] == ["materials.zip!/guide.md"]
|
||||
assert scan.skipped_code_count == 1
|
||||
assert scan.blocked_files == [
|
||||
{
|
||||
"path": "materials.zip!/../escape.md",
|
||||
"reason": "压缩包包含路径穿越或符号链接",
|
||||
}
|
||||
]
|
||||
assert "code-must-not-be-read" not in str(scan.manifest)
|
||||
@@ -0,0 +1,429 @@
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, UploadFile
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.routers import wiki_packages
|
||||
from deerflow.assistant_knowledge.archive import PackageArchive
|
||||
from deerflow.assistant_knowledge.package_io import write_package
|
||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
|
||||
from deerflow.integrations.weknora.local_index import embedding as embedding_module
|
||||
from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_vectors_and_links_survive_roundtrip_without_encoding(tmp_path, monkeypatch):
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
blob, _ = encode_vector([0.3, 0.5, 0.7])
|
||||
pages = [
|
||||
{
|
||||
"slug": "concept/cooling",
|
||||
"title": "散热",
|
||||
"page_type": "concept",
|
||||
"content": "散热管理用于降低设备温度。参见 [[entity/fan]]",
|
||||
"summary": "设备散热",
|
||||
"tags": ["设备", "维护"],
|
||||
"category_path": ["运维", "硬件"],
|
||||
"aliases": ["降温"],
|
||||
"out_links": ["entity/fan"],
|
||||
"in_links": [],
|
||||
},
|
||||
{"slug": "entity/fan", "title": "风扇", "page_type": "entity", "content": "风扇通过气流帮助散热,参见 [[concept/cooling]]。", "aliases": [], "out_links": ["concept/cooling"], "in_links": ["concept/cooling"]},
|
||||
]
|
||||
vectors = [
|
||||
{
|
||||
"wiki_slug": p["slug"],
|
||||
"section_index": 0,
|
||||
"section_content": p["content"],
|
||||
"content_hash": "test",
|
||||
"embedding_model": "test-model",
|
||||
"embedding_fingerprint": "test-fp",
|
||||
"embedding_dimensions": 3,
|
||||
"vector_blob_base64": base64.b64encode(blob).decode(),
|
||||
}
|
||||
for p in pages
|
||||
]
|
||||
original = tmp_path / "original.jsonl"
|
||||
entities = [{"id": p["slug"], "name": p["title"], "type": p["page_type"], "source_wiki_slug": p["slug"]} for p in pages]
|
||||
relations = [{"source_entity_id": pages[0]["slug"], "target_entity_id": pages[1]["slug"], "source_wiki_slug": pages[0]["slug"], "predicate": "references"}]
|
||||
write_package(original, [
|
||||
("source", {"provider": "weknora", "package_version": 1}),
|
||||
*[("wiki_page", p) for p in pages],
|
||||
*[("vector", v) for v in vectors],
|
||||
*[("entity", e) for e in entities],
|
||||
*[("relation", r) for r in relations],
|
||||
("chunk", {"id": "raw-chunk", "content": "原始文档分段不是 Wiki 页面"}),
|
||||
])
|
||||
archive = PackageArchive(original)
|
||||
base = await store.initialize(created_by="admin")
|
||||
job = await store.import_package(package=archive, source_type="weknora", source_key="source", source_name="来源", trigger="test", created_by="admin")
|
||||
assert job["counts"]["vectors"] == 2
|
||||
assert job["counts"]["wiki_pages"] == 2
|
||||
assert job["counts"]["chunk_pages"] == 1
|
||||
imported = await store.get_page(pages[0]["slug"])
|
||||
for field in ("category_path", "tags", "aliases", "out_links"):
|
||||
assert imported[field] == pages[0][field]
|
||||
assert imported["content_markdown"] == pages[0]["content"]
|
||||
# The global base may receive the same Wiki through multiple libraries.
|
||||
# Only the current article's import supplies vectors; older contributions
|
||||
# remain auditable but must not duplicate/stale the restored index.
|
||||
await store.import_package(package=archive, source_type="weknora", source_key="second-source", source_name="第二来源", trigger="test", created_by="admin")
|
||||
exported = tmp_path / "export.jsonl"
|
||||
write_package(exported, [record async for record in store.iter_export_records(base["id"])])
|
||||
restored_archive = PackageArchive(exported)
|
||||
assert {v["vector_blob_base64"] for v in restored_archive["vectors"]} == {base64.b64encode(blob).decode()}
|
||||
assert sorted((v["wiki_slug"], v["section_index"]) for v in restored_archive["vectors"]) == sorted((v["wiki_slug"], v["section_index"]) for v in vectors)
|
||||
assert all(r["source_wiki_slug"] in {p["slug"] for p in pages} for r in restored_archive["relations"])
|
||||
index = MemoryLlmWikiIndexStore()
|
||||
received = []
|
||||
|
||||
class Remote:
|
||||
async def create_wiki_folder(self, kb, *, name, parent_id=""):
|
||||
return {"id": f"{parent_id}/{name}"}
|
||||
|
||||
async def create_wiki_page(self, kb, page):
|
||||
received.append(page)
|
||||
return page
|
||||
|
||||
async def rebuild_wiki_links(self, kb):
|
||||
pass
|
||||
|
||||
class Embedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
raise AssertionError("Existing package must not be encoded")
|
||||
|
||||
app = FastAPI()
|
||||
app.state.llmwiki_index_store = index
|
||||
app.state.llmwiki_embedding = Embedding()
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.config = None
|
||||
monkeypatch.setattr(wiki_packages, "get_resolved_llmwiki_runtime", lambda c: None)
|
||||
monkeypatch.setattr(wiki_packages, "build_weknora_client", lambda c: Remote())
|
||||
monkeypatch.setattr(wiki_packages, "_package_root", lambda: tmp_path)
|
||||
from app.gateway import knowledge_transfer
|
||||
|
||||
async def sync(app, mapping):
|
||||
return {"status": "completed"}
|
||||
|
||||
monkeypatch.setattr(knowledge_transfer, "sync_to_global", sync)
|
||||
await wiki_packages.restore_package(app, {"id": "target", "weknora_id": "remote"}, "job-test", exported)
|
||||
for raw in received:
|
||||
assert raw["content"] == next(p["content"] for p in pages if p["slug"] == raw["slug"])
|
||||
local = await index.get_page("target", raw["slug"])
|
||||
assert local["content_hash"] == normalize_wiki_page("target", raw)["content_hash"]
|
||||
snapshot = await index.load_vector_snapshot("target", "test-fp")
|
||||
assert len(snapshot["rows"]) == 2
|
||||
assert all(r["vector_blob"] == blob for r in snapshot["rows"])
|
||||
|
||||
class SearchEmbedding:
|
||||
dimensions = 3
|
||||
fingerprint = "test-fp"
|
||||
|
||||
async def embed_query(self, query):
|
||||
return [1, 0, 0]
|
||||
|
||||
found = await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding())
|
||||
assert found and all(r["retrieval_mode"] == "vector" for r in found)
|
||||
SearchEmbedding.fingerprint = "different-model-same-dimension"
|
||||
assert await store.search_pages(base_ids=[base["id"]], query="没有关键词重合的查询", embedding_client=SearchEmbedding()) == []
|
||||
archive.close()
|
||||
restored_archive.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_is_bounded_cpu_and_does_not_need_http(monkeypatch, tmp_path):
|
||||
observed = {}
|
||||
|
||||
class LocalModel:
|
||||
@staticmethod
|
||||
def list_supported_models():
|
||||
return [{"model": "BAAI/bge-m3"}]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
observed.update(kwargs)
|
||||
|
||||
def embed(self, texts, *, batch_size):
|
||||
observed["batch_size"] = batch_size
|
||||
return [[1.0, *([0.0] * 1023)] for _ in texts]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "fastembed", SimpleNamespace(TextEmbedding=LocalModel))
|
||||
(tmp_path / "onnx").mkdir()
|
||||
(tmp_path / "onnx" / "model.onnx").touch()
|
||||
config = LocalWikiEmbeddingConfig(
|
||||
provider="local",
|
||||
model="bge-embedding-m3",
|
||||
dimensions=1024,
|
||||
batch_size=2,
|
||||
local_threads=2,
|
||||
local_model_path=str(tmp_path),
|
||||
local_process_isolation=False,
|
||||
)
|
||||
client = StrictWikiEmbeddingClient(config)
|
||||
assert len(await client.embed_query("本地查询")) == 1024
|
||||
assert observed["model_name"] == "BAAI/bge-m3"
|
||||
assert observed["providers"] == ["CPUExecutionProvider"]
|
||||
assert observed["local_files_only"] is True
|
||||
assert observed["specific_model_path"] == str(tmp_path)
|
||||
assert observed["batch_size"] == observed["threads"] == 2
|
||||
assert client._client is None
|
||||
assert client.local_encoder_status()["state"] == "ready"
|
||||
assert client.local_encoder_status()["model_files_ready"] is True
|
||||
|
||||
cold_client = StrictWikiEmbeddingClient(config)
|
||||
status = cold_client.start_local_encoder()
|
||||
assert status["state"] == "starting"
|
||||
assert status["running"] is True
|
||||
await cold_client._local_warmup_task
|
||||
assert cold_client.local_encoder_status()["state"] == "ready"
|
||||
|
||||
monkeypatch.delenv("DEERFLOW_BGE_M3_MODEL_PATH", raising=False)
|
||||
missing_path = StrictWikiEmbeddingClient(
|
||||
LocalWikiEmbeddingConfig(provider="local", model="bge-embedding-m3", dimensions=1024),
|
||||
)
|
||||
missing_path.start_local_encoder()
|
||||
await missing_path._local_warmup_task
|
||||
assert missing_path.local_encoder_status()["state"] == "failed"
|
||||
assert "目录未配置" in str(missing_path.local_encoder_status()["last_error"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_encoder_uses_offline_isolated_worker(monkeypatch, tmp_path):
|
||||
(tmp_path / "onnx").mkdir()
|
||||
(tmp_path / "onnx" / "model.onnx").touch()
|
||||
observed: dict = {}
|
||||
|
||||
class FakeStdout:
|
||||
def __init__(self):
|
||||
self.lines = [json.dumps({"status": "ready"}) + "\n"]
|
||||
|
||||
def readline(self):
|
||||
return self.lines.pop(0) if self.lines else ""
|
||||
|
||||
class FakeProcess:
|
||||
def __init__(self):
|
||||
self.stdout = FakeStdout()
|
||||
self.returncode = None
|
||||
self.stdin = SimpleNamespace(write=self.write, flush=lambda: None)
|
||||
|
||||
def write(self, line):
|
||||
payload = json.loads(line)
|
||||
if payload.get("command") == "shutdown":
|
||||
self.returncode = 0
|
||||
return
|
||||
self.stdout.lines.append(
|
||||
json.dumps(
|
||||
{
|
||||
"id": payload["id"],
|
||||
"status": "ok",
|
||||
"vectors": [[1.0, *([0.0] * 1023)] for _ in payload["texts"]],
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return self.returncode
|
||||
|
||||
def terminate(self):
|
||||
self.returncode = -15
|
||||
|
||||
def kill(self):
|
||||
self.returncode = -9
|
||||
|
||||
process = FakeProcess()
|
||||
|
||||
def fake_popen(command, **kwargs):
|
||||
observed.update({"command": command, **kwargs})
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(embedding_module.subprocess, "Popen", fake_popen)
|
||||
client = StrictWikiEmbeddingClient(
|
||||
LocalWikiEmbeddingConfig(
|
||||
provider="local",
|
||||
model="bge-embedding-m3",
|
||||
dimensions=1024,
|
||||
local_model_path=str(tmp_path),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(await client.embed_query("隔离编码")) == 1024
|
||||
assert "deerflow.integrations.weknora.local_index.embedding_worker" in observed["command"]
|
||||
assert observed["env"]["HF_HUB_OFFLINE"] == "1"
|
||||
assert observed["env"]["PYTHONIOENCODING"] == "utf-8"
|
||||
assert client.local_encoder_status()["process_isolation"] is True
|
||||
await client.close()
|
||||
assert process.poll() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_builtin_skill_without_ownership_record_can_be_distilled(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def admin(_):
|
||||
return "admin", True
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, *args, **kwargs):
|
||||
return {"name": "测试库", "weknora_id": "remote"}
|
||||
|
||||
class Jobs:
|
||||
async def create_job(self, **kwargs):
|
||||
return kwargs
|
||||
|
||||
app = FastAPI()
|
||||
app.state.config = None
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
# Deliberately no skill_store ownership row: built-ins live on disk.
|
||||
monkeypatch.setattr(skill_knowledge, "_actor", admin)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
body = skill_knowledge.CreateSyncJobRequest(skill_names=["builtin"], targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}])
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
assert job["skill_names"] == ["builtin"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_regular_user_can_distill_visible_skill_only_to_writable_wiki(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import skill_knowledge
|
||||
|
||||
async def regular_user(_):
|
||||
return "user-1", False
|
||||
|
||||
class Skills:
|
||||
async def get_visible(self, name, user_id):
|
||||
assert user_id == "user-1"
|
||||
return {"name": name} if name == "visible-skill" else None
|
||||
|
||||
async def list_favorite_skill_names(self, user_id):
|
||||
assert user_id == "user-1"
|
||||
return []
|
||||
|
||||
class Mappings:
|
||||
async def get_authorized(self, mapping_id, user_id, *, write, is_admin):
|
||||
assert (mapping_id, user_id, write, is_admin) == ("mapping", "user-1", True, False)
|
||||
return {"name": "我的 Wiki", "weknora_id": "remote"}
|
||||
|
||||
class Jobs:
|
||||
async def create_job(self, **kwargs):
|
||||
return kwargs
|
||||
|
||||
app = FastAPI()
|
||||
app.state.config = None
|
||||
app.state.skill_store = Skills()
|
||||
app.state.llmwiki_store = Mappings()
|
||||
app.state.skill_knowledge_store = Jobs()
|
||||
monkeypatch.setattr(skill_knowledge, "_actor", regular_user)
|
||||
monkeypatch.setattr(skill_knowledge, "_skill_directory", lambda *args: tmp_path)
|
||||
monkeypatch.setattr(
|
||||
skill_knowledge,
|
||||
"get_or_new_skill_storage",
|
||||
lambda **_: SimpleNamespace(get_skills_root_path=lambda: tmp_path),
|
||||
)
|
||||
body = skill_knowledge.CreateSyncJobRequest(
|
||||
skill_names=["visible-skill"],
|
||||
targets=[{"target_type": "weknora", "target_mode": "wiki", "target_id": "mapping"}],
|
||||
review_mode="required",
|
||||
)
|
||||
|
||||
job = await skill_knowledge.create_sync_job(Request({"type": "http", "app": app}), body)
|
||||
|
||||
assert job["created_by"] == "user-1"
|
||||
assert job["review_mode"] == "off"
|
||||
|
||||
|
||||
def test_short_wiki_summary_and_body_still_have_a_vector_section():
|
||||
sections = chunk_wiki_page({"title": "风扇", "summary": "通过气流帮助散热。", "content_md": "# 风扇\n用于降低设备温度。"}, max_chars=1000, overlap=100, min_chars=80, title_prefix_enabled=True)
|
||||
assert len(sections) == 1
|
||||
assert "降低设备温度" in sections[0].embedding_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_can_upload_and_activate_offline_encoder(monkeypatch, tmp_path):
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import llmwiki_index
|
||||
from deerflow.config.system_settings import SystemSettings
|
||||
|
||||
async def admin(_):
|
||||
return "admin", True
|
||||
|
||||
class Embedding:
|
||||
def __init__(self):
|
||||
self.model_path = ""
|
||||
|
||||
async def reset_local_encoder(self, *, model_path=None):
|
||||
if model_path is not None:
|
||||
self.model_path = model_path
|
||||
|
||||
def local_encoder_status(self):
|
||||
return {
|
||||
"provider": "local",
|
||||
"model": "bge-embedding-m3",
|
||||
"state": "idle",
|
||||
"model_path": self.model_path,
|
||||
"model_files_ready": bool(self.model_path),
|
||||
}
|
||||
|
||||
saved = []
|
||||
monkeypatch.setattr(llmwiki_index, "_actor", admin)
|
||||
monkeypatch.setattr(llmwiki_index, "runtime_home", lambda: tmp_path)
|
||||
monkeypatch.setattr(llmwiki_index, "load_system_settings", SystemSettings)
|
||||
monkeypatch.setattr(llmwiki_index, "save_system_settings", lambda value: saved.append(value) or value)
|
||||
embedding = Embedding()
|
||||
app = FastAPI()
|
||||
app.state.config = SimpleNamespace(
|
||||
llmwiki=SimpleNamespace(
|
||||
local_wiki_index=SimpleNamespace(embedding=SimpleNamespace(provider="local"))
|
||||
)
|
||||
)
|
||||
app.state.llmwiki_embedding = embedding
|
||||
request = Request({"type": "http", "headers": [], "app": app})
|
||||
files = [
|
||||
UploadFile(filename="model.onnx", file=BytesIO(b"onnx")),
|
||||
UploadFile(filename="config.json", file=BytesIO(b"{}")),
|
||||
]
|
||||
|
||||
result = await llmwiki_index.upload_local_encoder(
|
||||
request,
|
||||
files=files,
|
||||
paths=["onnx/model.onnx", "config.json"],
|
||||
)
|
||||
|
||||
active = tmp_path / "models" / "bge-m3"
|
||||
assert (active / "onnx" / "model.onnx").read_bytes() == b"onnx"
|
||||
assert result["uploaded_files"] == 2
|
||||
assert saved[-1].local_wiki_encoder.model_path == str(active.resolve())
|
||||
assert embedding.model_path == str(active.resolve())
|
||||
|
||||
|
||||
def test_encoder_upload_rejects_parent_traversal():
|
||||
from app.gateway.routers import llmwiki_index
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
llmwiki_index._safe_relative_upload_path("../model.onnx")
|
||||
assert getattr(exc_info.value, "status_code", None) == 422
|
||||
Reference in new issue
Block a user