sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -9,7 +9,12 @@ from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_p
|
||||
from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
@@ -20,23 +25,36 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
await connection.run_sync(
|
||||
lambda sync_connection: Base.metadata.create_all(
|
||||
sync_connection,
|
||||
tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiSyncStateRow.__table__],
|
||||
tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiEdgeRow.__table__, LlmWikiWikiSyncStateRow.__table__],
|
||||
)
|
||||
)
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with session_factory() as session:
|
||||
session.add(
|
||||
LlmWikiKnowledgeBaseRow(
|
||||
id="mapping-1",
|
||||
weknora_id="remote-1",
|
||||
owner_user_id="u1",
|
||||
name="Wiki",
|
||||
description="",
|
||||
kb_type="document",
|
||||
publication_status="private",
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
session.add_all(
|
||||
[
|
||||
LlmWikiKnowledgeBaseRow(
|
||||
id="mapping-1",
|
||||
weknora_id="remote-1",
|
||||
owner_user_id="u1",
|
||||
name="Wiki",
|
||||
description="",
|
||||
kb_type="document",
|
||||
publication_status="private",
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
),
|
||||
LlmWikiKnowledgeBaseRow(
|
||||
id="mapping-2",
|
||||
weknora_id="remote-2",
|
||||
owner_user_id="u2",
|
||||
name="Private Wiki",
|
||||
description="",
|
||||
kb_type="document",
|
||||
publication_status="private",
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
),
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
@@ -86,7 +104,165 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
||||
assert page_statuses[0]["indexed_at"]
|
||||
assert page_statuses[0]["updated_at"]
|
||||
|
||||
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
|
||||
second_page = normalize_wiki_page(
|
||||
"mapping-1",
|
||||
{
|
||||
"id": "p2",
|
||||
"slug": "companies/acme",
|
||||
"title": "Acme",
|
||||
"content": "Acme is connected to Alice.",
|
||||
"status": "published",
|
||||
},
|
||||
)
|
||||
second = await store.replace_page_vectors(
|
||||
page=second_page,
|
||||
vectors=[
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": "Acme",
|
||||
"section_content": "Acme is connected to Alice.",
|
||||
"content_hash": "acme-section",
|
||||
"embedding_dimensions": dimensions,
|
||||
"vector_blob": blob,
|
||||
}
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="sync-1",
|
||||
)
|
||||
foreign_page = normalize_wiki_page(
|
||||
"mapping-2",
|
||||
{
|
||||
"id": "foreign-p1",
|
||||
"slug": "private/foreign",
|
||||
"title": "Foreign",
|
||||
"content": "This page belongs to another mapping.",
|
||||
"status": "published",
|
||||
},
|
||||
)
|
||||
await store.replace_page_vectors(
|
||||
page=foreign_page,
|
||||
vectors=[
|
||||
{
|
||||
"section_index": 0,
|
||||
"heading": "Foreign",
|
||||
"section_content": "This page belongs to another mapping.",
|
||||
"content_hash": "foreign-section",
|
||||
"embedding_dimensions": dimensions,
|
||||
"vector_blob": blob,
|
||||
}
|
||||
],
|
||||
fingerprint="fp",
|
||||
sync_id="sync-foreign",
|
||||
)
|
||||
stale_page = normalize_wiki_page(
|
||||
"mapping-1",
|
||||
{
|
||||
"id": "p1",
|
||||
"slug": "people/alice",
|
||||
"title": "Stale Alice",
|
||||
"content": "A stale worker must not replace this page.",
|
||||
"status": "published",
|
||||
},
|
||||
)
|
||||
assert not await store.update_sync_progress(
|
||||
"mapping-1",
|
||||
owner="worker-stale",
|
||||
sync_id="sync-stale",
|
||||
remote_page_count=999,
|
||||
)
|
||||
assert not await store.touch_page(
|
||||
str(saved["id"]),
|
||||
"sync-stale",
|
||||
mapping_id="mapping-1",
|
||||
owner="worker-stale",
|
||||
)
|
||||
assert await store.replace_page_vectors(
|
||||
page=stale_page,
|
||||
vectors=[],
|
||||
fingerprint="fp",
|
||||
sync_id="sync-stale",
|
||||
owner="worker-stale",
|
||||
) is None
|
||||
assert not await store.record_page_failure(
|
||||
page=stale_page,
|
||||
sync_id="sync-stale",
|
||||
error="stale failure",
|
||||
owner="worker-stale",
|
||||
)
|
||||
preserved = await store.get_page("mapping-1", "people/alice")
|
||||
assert preserved is not None
|
||||
assert preserved["title"] == "Alice"
|
||||
assert preserved["last_seen_sync_id"] == "sync-1"
|
||||
assert preserved["last_error"] is None
|
||||
assert await store.replace_graph_edges(
|
||||
"mapping-1",
|
||||
[],
|
||||
owner="worker-stale",
|
||||
sync_id="sync-stale",
|
||||
) is None
|
||||
assert await store.mark_missing_pages_deleted(
|
||||
"mapping-1",
|
||||
"sync-stale",
|
||||
owner="worker-stale",
|
||||
) is None
|
||||
|
||||
graph = await store.replace_graph_edges(
|
||||
"mapping-1",
|
||||
[
|
||||
{
|
||||
"source_slug": saved["slug"],
|
||||
"target_slug": second["slug"],
|
||||
"edge_type": "wiki_link",
|
||||
},
|
||||
{
|
||||
"source_slug": saved["slug"],
|
||||
"target_slug": foreign_page["slug"],
|
||||
"edge_type": "wiki_link",
|
||||
},
|
||||
],
|
||||
owner="worker",
|
||||
sync_id="sync-1",
|
||||
)
|
||||
assert graph == {"edge_count": 1, "changed": True}
|
||||
assert await store.replace_graph_edges(
|
||||
"mapping-1",
|
||||
[
|
||||
{
|
||||
"source_slug": saved["slug"],
|
||||
"target_slug": second["slug"],
|
||||
"edge_type": "wiki_link",
|
||||
}
|
||||
],
|
||||
owner="worker",
|
||||
sync_id="sync-1",
|
||||
) == {"edge_count": 1, "changed": False}
|
||||
outgoing = await store.get_adjacent_pages(["mapping-1"], {"mapping-1": [saved["slug"]]}, direction="outgoing")
|
||||
assert len(outgoing) == 1
|
||||
assert outgoing[0]["neighbor_slug"] == second["slug"]
|
||||
assert outgoing[0]["page"]["knowledge_base_mapping_id"] == "mapping-1"
|
||||
incoming = await store.get_adjacent_pages(["mapping-1"], {"mapping-1": [second["slug"]]}, direction="incoming")
|
||||
assert len(incoming) == 1
|
||||
assert incoming[0]["neighbor_slug"] == saved["slug"]
|
||||
|
||||
await store.finish_sync(
|
||||
"mapping-1",
|
||||
owner="worker",
|
||||
sync_id="sync-1",
|
||||
success=True,
|
||||
remote_page_count=2,
|
||||
)
|
||||
assert await store.try_acquire_sync_lease(
|
||||
"mapping-1",
|
||||
owner="worker-new",
|
||||
sync_id="sync-2",
|
||||
lease_seconds=60,
|
||||
fingerprint="fp",
|
||||
)
|
||||
assert await store.mark_missing_pages_deleted(
|
||||
"mapping-1",
|
||||
"sync-2",
|
||||
owner="worker-new",
|
||||
) == 2
|
||||
assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
Reference in new issue
Block a user