sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -9,7 +9,12 @@ from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_p
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector
from deerflow.persistence.base import Base
from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
@@ -20,23 +25,36 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
await connection.run_sync(
lambda sync_connection: Base.metadata.create_all(
sync_connection,
tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiSyncStateRow.__table__],
tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiEdgeRow.__table__, LlmWikiWikiSyncStateRow.__table__],
)
)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
async with session_factory() as session:
session.add(
LlmWikiKnowledgeBaseRow(
id="mapping-1",
weknora_id="remote-1",
owner_user_id="u1",
name="Wiki",
description="",
kb_type="document",
publication_status="private",
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
session.add_all(
[
LlmWikiKnowledgeBaseRow(
id="mapping-1",
weknora_id="remote-1",
owner_user_id="u1",
name="Wiki",
description="",
kb_type="document",
publication_status="private",
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
),
LlmWikiKnowledgeBaseRow(
id="mapping-2",
weknora_id="remote-2",
owner_user_id="u2",
name="Private Wiki",
description="",
kb_type="document",
publication_status="private",
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
),
]
)
await session.commit()
@@ -86,7 +104,165 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
assert page_statuses[0]["indexed_at"]
assert page_statuses[0]["updated_at"]
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
second_page = normalize_wiki_page(
"mapping-1",
{
"id": "p2",
"slug": "companies/acme",
"title": "Acme",
"content": "Acme is connected to Alice.",
"status": "published",
},
)
second = await store.replace_page_vectors(
page=second_page,
vectors=[
{
"section_index": 0,
"heading": "Acme",
"section_content": "Acme is connected to Alice.",
"content_hash": "acme-section",
"embedding_dimensions": dimensions,
"vector_blob": blob,
}
],
fingerprint="fp",
sync_id="sync-1",
)
foreign_page = normalize_wiki_page(
"mapping-2",
{
"id": "foreign-p1",
"slug": "private/foreign",
"title": "Foreign",
"content": "This page belongs to another mapping.",
"status": "published",
},
)
await store.replace_page_vectors(
page=foreign_page,
vectors=[
{
"section_index": 0,
"heading": "Foreign",
"section_content": "This page belongs to another mapping.",
"content_hash": "foreign-section",
"embedding_dimensions": dimensions,
"vector_blob": blob,
}
],
fingerprint="fp",
sync_id="sync-foreign",
)
stale_page = normalize_wiki_page(
"mapping-1",
{
"id": "p1",
"slug": "people/alice",
"title": "Stale Alice",
"content": "A stale worker must not replace this page.",
"status": "published",
},
)
assert not await store.update_sync_progress(
"mapping-1",
owner="worker-stale",
sync_id="sync-stale",
remote_page_count=999,
)
assert not await store.touch_page(
str(saved["id"]),
"sync-stale",
mapping_id="mapping-1",
owner="worker-stale",
)
assert await store.replace_page_vectors(
page=stale_page,
vectors=[],
fingerprint="fp",
sync_id="sync-stale",
owner="worker-stale",
) is None
assert not await store.record_page_failure(
page=stale_page,
sync_id="sync-stale",
error="stale failure",
owner="worker-stale",
)
preserved = await store.get_page("mapping-1", "people/alice")
assert preserved is not None
assert preserved["title"] == "Alice"
assert preserved["last_seen_sync_id"] == "sync-1"
assert preserved["last_error"] is None
assert await store.replace_graph_edges(
"mapping-1",
[],
owner="worker-stale",
sync_id="sync-stale",
) is None
assert await store.mark_missing_pages_deleted(
"mapping-1",
"sync-stale",
owner="worker-stale",
) is None
graph = await store.replace_graph_edges(
"mapping-1",
[
{
"source_slug": saved["slug"],
"target_slug": second["slug"],
"edge_type": "wiki_link",
},
{
"source_slug": saved["slug"],
"target_slug": foreign_page["slug"],
"edge_type": "wiki_link",
},
],
owner="worker",
sync_id="sync-1",
)
assert graph == {"edge_count": 1, "changed": True}
assert await store.replace_graph_edges(
"mapping-1",
[
{
"source_slug": saved["slug"],
"target_slug": second["slug"],
"edge_type": "wiki_link",
}
],
owner="worker",
sync_id="sync-1",
) == {"edge_count": 1, "changed": False}
outgoing = await store.get_adjacent_pages(["mapping-1"], {"mapping-1": [saved["slug"]]}, direction="outgoing")
assert len(outgoing) == 1
assert outgoing[0]["neighbor_slug"] == second["slug"]
assert outgoing[0]["page"]["knowledge_base_mapping_id"] == "mapping-1"
incoming = await store.get_adjacent_pages(["mapping-1"], {"mapping-1": [second["slug"]]}, direction="incoming")
assert len(incoming) == 1
assert incoming[0]["neighbor_slug"] == saved["slug"]
await store.finish_sync(
"mapping-1",
owner="worker",
sync_id="sync-1",
success=True,
remote_page_count=2,
)
assert await store.try_acquire_sync_lease(
"mapping-1",
owner="worker-new",
sync_id="sync-2",
lease_seconds=60,
fingerprint="fp",
)
assert await store.mark_missing_pages_deleted(
"mapping-1",
"sync-2",
owner="worker-new",
) == 2
assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
await engine.dispose()