feat: improve knowledge sync and entity alignment
This commit is contained in:
1 parent
7dbac9cc56
commit
dea4697ea0
38 files changed
+3742
-634
No files matched your search
@@ -3,6 +3,7 @@ import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.llmwiki_rag import _source_from_assistant_result, _source_from_result
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
@@ -194,3 +195,264 @@ async def test_imported_vectors_are_reused_for_assistant_search(
|
||||
query="这个查询不靠关键词",
|
||||
embedding_client=FakeEmbeddingClient(),
|
||||
) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reimport_retires_pages_removed_from_source_snapshot(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
common = {
|
||||
"source_type": "weknora",
|
||||
"source_key": "kb-refresh",
|
||||
"source_name": "自动同步普通库",
|
||||
"trigger": "vectorization_completed",
|
||||
"created_by": "admin",
|
||||
"mapping_id": "kb-refresh",
|
||||
}
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "保留内容"},
|
||||
{"slug": "wiki/removed", "title": "删除页", "content": "旧内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/kept", "title": "保留页", "content": "新内容"},
|
||||
],
|
||||
},
|
||||
**common,
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"], include_chunks=True)
|
||||
assert [page["slug"] for page in pages] == ["wiki/kept"]
|
||||
assert pages[0]["content_markdown"] == "新内容"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_base_can_be_renamed_and_deleted_without_affecting_global_base(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
global_base = await repository.initialize(created_by="admin")
|
||||
custom_base = await repository.create_base(
|
||||
created_by="admin",
|
||||
name="待编辑助手库",
|
||||
description="旧描述",
|
||||
)
|
||||
await repository.import_package(
|
||||
package={
|
||||
"source": {"type": "weknora"},
|
||||
"wiki_pages": [
|
||||
{"slug": "wiki/delete-test", "title": "删除测试", "content": "保留在总库中的内容"}
|
||||
],
|
||||
"entities": [{"local_id": "e1", "type": "concept", "name": "删除测试实体"}],
|
||||
"relations": [],
|
||||
},
|
||||
source_type="weknora",
|
||||
source_key="kb-delete-test",
|
||||
source_name="删除测试普通库",
|
||||
trigger="manual_from_weknora",
|
||||
created_by="admin",
|
||||
mapping_id="kb-delete-test",
|
||||
base_id=custom_base["id"],
|
||||
)
|
||||
|
||||
updated = await repository.update_base(
|
||||
custom_base["id"],
|
||||
name="已编辑助手库",
|
||||
description="新描述",
|
||||
)
|
||||
assert updated is not None
|
||||
assert updated["name"] == "已编辑助手库"
|
||||
assert updated["description"] == "新描述"
|
||||
|
||||
assert await repository.delete_base(custom_base["id"])
|
||||
assert await repository.get_base(custom_base["id"]) is None
|
||||
assert all(base["id"] != custom_base["id"] for base in await repository.list_bases())
|
||||
assert await repository.list_pages(base_id=global_base["id"], query="保留在总库")
|
||||
with pytest.raises(ValueError, match="不能删除"):
|
||||
await repository.delete_base(global_base["id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_global_base_aligns_same_entity_title_and_searches_knowledge_body(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
first_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-one", "knowledge_base_name": "人员库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "people/zhang-san",
|
||||
"title": "张 三",
|
||||
"page_type": "article",
|
||||
"content": "张三负责设备巡检。",
|
||||
"summary": "摘要专用词:月度综述。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "one",
|
||||
"type": "人物",
|
||||
"name": "张 三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "people/zhang-san",
|
||||
}
|
||||
],
|
||||
}
|
||||
second_package = {
|
||||
"source": {"provider": "weknora", "knowledge_base_id": "kb-two", "knowledge_base_name": "职责库"},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "entity/person-001",
|
||||
"title": "Zhang San",
|
||||
"page_type": "concept",
|
||||
"content": "张三负责系统故障处置。",
|
||||
"summary": "另一条摘要。",
|
||||
}
|
||||
],
|
||||
"entities": [
|
||||
{
|
||||
"local_id": "two",
|
||||
"type": "person",
|
||||
"name": "张三",
|
||||
"aliases": ["Zhang San"],
|
||||
"source_wiki_slug": "entity/person-001",
|
||||
}
|
||||
],
|
||||
}
|
||||
for package, source_key in ((first_package, "kb-one"), (second_package, "kb-two")):
|
||||
await repository.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=source_key,
|
||||
source_name=source_key,
|
||||
trigger="vectorization_completed",
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert "张三负责设备巡检。" in pages[0]["content_markdown"]
|
||||
assert "张三负责系统故障处置。" in pages[0]["content_markdown"]
|
||||
assert len(pages[0]["origins"]) == 2
|
||||
|
||||
assert await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="月度综述",
|
||||
embedding_client=None,
|
||||
) == []
|
||||
results = await repository.search_pages(
|
||||
base_ids=[base["id"]],
|
||||
query="系统故障处置",
|
||||
embedding_client=None,
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert "系统故障处置" in results[0]["matched_section"]["content"]
|
||||
assert "月度综述" not in results[0]["matched_section"]["content"]
|
||||
|
||||
async with repository._sf() as session:
|
||||
entities = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityRow).where(
|
||||
AssistantKnowledgeEntityRow.base_id == base["id"],
|
||||
AssistantKnowledgeEntityRow.status == "active",
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(entities) == 1
|
||||
contributions = (
|
||||
await session.execute(
|
||||
select(AssistantKnowledgeEntityContributionRow).where(
|
||||
AssistantKnowledgeEntityContributionRow.entity_id == entities[0].id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(contributions) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_uploaded_package_reuses_embedded_weknora_source_identity(
|
||||
repository: AssistantKnowledgeRepository,
|
||||
) -> None:
|
||||
base = await repository.initialize(created_by="admin")
|
||||
for upload_no, content in ((1, "旧知识"), (2, "新知识")):
|
||||
job = await repository.create_queued_import_job(
|
||||
source_type="weknora",
|
||||
source_key=f"upload:{upload_no}",
|
||||
source_name=f"upload-{upload_no}.jsonl",
|
||||
trigger="manual_package_upload",
|
||||
created_by="admin",
|
||||
base_id=base["id"],
|
||||
)
|
||||
await repository.import_package_for_job(
|
||||
job["id"],
|
||||
package={
|
||||
"source": {
|
||||
"provider": "weknora",
|
||||
"knowledge_base_id": "stable-kb-id",
|
||||
"knowledge_base_name": "同一个普通知识库",
|
||||
},
|
||||
"wiki_pages": [
|
||||
{
|
||||
"slug": "wiki/stable",
|
||||
"title": "稳定实体",
|
||||
"content": content,
|
||||
}
|
||||
],
|
||||
},
|
||||
created_by="admin",
|
||||
)
|
||||
|
||||
active_sources = await repository.list_sources(base_id=base["id"])
|
||||
assert len(active_sources) == 1
|
||||
assert active_sources[0]["source_key"] == "stable-kb-id"
|
||||
pages = await repository.list_pages(base_id=base["id"])
|
||||
assert len(pages) == 1
|
||||
assert pages[0]["content_markdown"] == "新知识"
|
||||
|
||||
|
||||
def test_assistant_citation_uses_matched_knowledge_instead_of_summary() -> None:
|
||||
source = _source_from_assistant_result(
|
||||
{
|
||||
"id": "page-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content_markdown": "完整页面正文",
|
||||
"matched_section": {
|
||||
"content": "实际命中的知识片段:设备过热时先断电。",
|
||||
},
|
||||
"score": 0.91,
|
||||
},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert source["snippet"] == "实际命中的知识片段:设备过热时先断电。"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
|
||||
|
||||
def test_regular_wiki_citation_does_not_fall_back_to_summary() -> None:
|
||||
source = _source_from_result(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"wiki_page_id": "wiki-1",
|
||||
"title": "设备知识",
|
||||
"summary": "只是一段摘要",
|
||||
"content": "实际知识正文",
|
||||
"matched_sections": [],
|
||||
},
|
||||
{"id": "kb-1", "name": "普通知识库"},
|
||||
1,
|
||||
)
|
||||
|
||||
assert source["content"] == "实际知识正文"
|
||||
assert source["snippet"] == "实际知识正文"
|
||||
assert "只是一段摘要" not in source["snippet"]
|
||||
Reference in new issue
Block a user