sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -15,7 +15,11 @@ class FakeEmbedding:
|
||||
dimensions = 2
|
||||
fingerprint = "fp"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.queries: list[str] = []
|
||||
|
||||
async def embed_query(self, text: str) -> list[float]:
|
||||
self.queries.append(text)
|
||||
return [1.0, 0.0]
|
||||
|
||||
|
||||
@@ -71,6 +75,87 @@ async def test_exact_search_groups_sections_by_wiki_page_and_filters_drafts() ->
|
||||
assert [item["wiki_slug"] for item in (await narrow_service.search("alice", [mapping]))["results"]] == ["alice"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_title_boost_recalls_and_reranks_exact_title_page() -> None:
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
|
||||
title_page = normalize_wiki_page(
|
||||
"kb",
|
||||
{"slug": "mar-a-lago", "title": "海湖庄园", "content": "庄园资料", "status": "published"},
|
||||
)
|
||||
broad_page = normalize_wiki_page(
|
||||
"kb",
|
||||
{"slug": "trump", "title": "特朗普", "content": "人物资料", "status": "published"},
|
||||
)
|
||||
title_vector, _ = encode_vector([0.8, 0.6], dimensions=2)
|
||||
broad_vector, _ = encode_vector([1.0, 0.0], dimensions=2)
|
||||
await store.replace_page_vectors(
|
||||
page=title_page,
|
||||
vectors=[
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"section_content": "庄园资料",
|
||||
"content_hash": "title",
|
||||
"embedding_dimensions": 2,
|
||||
"vector_blob": title_vector,
|
||||
}
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="s",
|
||||
)
|
||||
await store.replace_page_vectors(
|
||||
page=broad_page,
|
||||
vectors=[
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": None,
|
||||
"section_content": "人物资料提到海湖庄园",
|
||||
"content_hash": "broad",
|
||||
"embedding_dimensions": 2,
|
||||
"vector_blob": broad_vector,
|
||||
}
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="s",
|
||||
)
|
||||
mapping = {"id": "kb", "name": "人物资料库"}
|
||||
|
||||
boosted = WikiVectorSearchService(
|
||||
LocalWikiIndexConfig(min_similarity=0.85, top_k_sections=2, top_k_pages=2),
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store, revision_check_seconds=0),
|
||||
)
|
||||
legacy = WikiVectorSearchService(
|
||||
LocalWikiIndexConfig(
|
||||
min_similarity=0.85,
|
||||
top_k_sections=2,
|
||||
top_k_pages=2,
|
||||
title_boost_enabled=False,
|
||||
),
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store, revision_check_seconds=0),
|
||||
)
|
||||
zero_weight = WikiVectorSearchService(
|
||||
LocalWikiIndexConfig(
|
||||
min_similarity=0.85,
|
||||
top_k_sections=2,
|
||||
top_k_pages=2,
|
||||
title_boost_enabled=True,
|
||||
title_boost_weight=0,
|
||||
),
|
||||
FakeEmbedding(),
|
||||
WikiVectorCache(store, revision_check_seconds=0),
|
||||
)
|
||||
|
||||
assert [item["wiki_slug"] for item in (await boosted.search("海湖庄园", [mapping]))["results"]] == [
|
||||
"mar-a-lago",
|
||||
"trump",
|
||||
]
|
||||
assert [item["wiki_slug"] for item in (await legacy.search("海湖庄园", [mapping]))["results"]] == ["trump"]
|
||||
assert [item["wiki_slug"] for item in (await zero_weight.search("海湖庄园", [mapping]))["results"]] == ["trump"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_corrupt_blob_marks_index_for_rebuild() -> None:
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
@@ -94,3 +179,56 @@ async def test_corrupt_blob_marks_index_for_rebuild() -> None:
|
||||
await service.search("broken", [{"id": "kb", "name": "Broken"}])
|
||||
assert exc_info.value.code == "WIKI_INDEX_REBUILDING"
|
||||
assert store.states["kb"]["state"] == "rebuild_required"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overview_query_uses_representative_body_without_similarity_filter_or_embedding() -> None:
|
||||
store = MemoryLlmWikiIndexStore()
|
||||
await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
|
||||
directory = normalize_wiki_page(
|
||||
"kb",
|
||||
{"slug": "index", "title": "目录", "page_type": "directory", "content": "目录内容", "status": "published"},
|
||||
)
|
||||
article = normalize_wiki_page(
|
||||
"kb",
|
||||
{
|
||||
"slug": "guides/cooling",
|
||||
"title": "设备散热",
|
||||
"page_type": "article",
|
||||
"summary": "摘要不应替代正文",
|
||||
"content": "设备过热时应先断电,再检查风道和散热风扇。",
|
||||
"status": "published",
|
||||
},
|
||||
)
|
||||
unrelated, _ = encode_vector([0.0, 1.0], dimensions=2)
|
||||
for page, content in ((directory, "目录内容"), (article, "设备过热时应先断电,再检查风道和散热风扇。")):
|
||||
await store.replace_page_vectors(
|
||||
page=page,
|
||||
vectors=[
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": "页面摘要" if page is article else None,
|
||||
"section_content": "摘要不应替代正文" if page is article else content,
|
||||
"content_hash": page["content_hash"],
|
||||
"embedding_dimensions": 2,
|
||||
"vector_blob": unrelated,
|
||||
}
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="s",
|
||||
)
|
||||
embedding = FakeEmbedding()
|
||||
service = WikiVectorSearchService(
|
||||
LocalWikiIndexConfig(min_similarity=0.99, top_k_sections=10, top_k_pages=8),
|
||||
embedding,
|
||||
WikiVectorCache(store, revision_check_seconds=0),
|
||||
)
|
||||
|
||||
result = await service.search("此知识库都讲了什么?", [{"id": "kb", "name": "设备库"}])
|
||||
|
||||
assert result["mode"] == "overview"
|
||||
assert [item["wiki_slug"] for item in result["results"]] == ["guides/cooling"]
|
||||
assert result["results"][0]["retrieval_mode"] == "overview"
|
||||
assert "设备过热时应先断电" in result["results"][0]["content"]
|
||||
assert "摘要不应替代正文" not in result["results"][0]["content"]
|
||||
assert embedding.queries == []
|
||||
Reference in new issue
Block a user