sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import aliased
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
def _now() -> datetime:
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.flush()
return row
async def _fence_current_sync_lease(
self,
session: AsyncSession,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
"""Fence destructive snapshot writes behind the current live lease."""
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
# ignores FOR UPDATE, while this statement obtains its write lock on
# every supported backend before the page/edge mutation starts.
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(updated_at=_now())
)
return bool((await session.execute(stmt)).rowcount)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
@@ -53,18 +85,351 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
if not normalized_ids or not normalized_tokens:
return []
conditions = []
for token in normalized_tokens[:32]:
needle = f"%{token}%"
conditions.extend(
(
func.lower(LlmWikiWikiPageRow.title).like(needle),
func.lower(LlmWikiWikiPageRow.summary).like(needle),
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
)
)
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
or_(*conditions),
)
.order_by(LlmWikiWikiPageRow.updated_at.desc())
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
await session.execute(
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(
dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip())
)
if not normalized_ids:
return []
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
)
.order_by(
LlmWikiWikiPageRow.remote_updated_at.desc(),
LlmWikiWikiPageRow.updated_at.desc(),
)
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Atomically replace an authorized mapping's resolved edge snapshot."""
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
page_stmt = select(LlmWikiWikiPageRow).where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
pages = list((await session.execute(page_stmt)).scalars())
pages_by_slug = {page.slug: page for page in pages}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
existing_stmt = select(
LlmWikiWikiEdgeRow.source_page_id,
LlmWikiWikiEdgeRow.target_page_id,
LlmWikiWikiEdgeRow.edge_type,
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
replacement_ids = {
(
pages_by_slug[source].id,
pages_by_slug[target].id,
edge_type,
)
for source, target, edge_type in canonical
}
changed = existing != replacement_ids
if changed:
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
now = _now()
edge_rows = [
{
"id": str(uuid4()),
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source].id,
"target_page_id": pages_by_slug[target].id,
"edge_type": edge_type,
"created_at": now,
}
for source, target, edge_type in sorted(canonical)
]
for start in range(0, len(edge_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiEdgeRow),
edge_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"edge_count": len(canonical), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
if not allowed:
return []
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
else:
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
seeds = {mapping_id: shared for mapping_id in allowed}
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
if not seeds:
return []
source_page = aliased(LlmWikiWikiPageRow)
target_page = aliased(LlmWikiWikiPageRow)
statuses = ["published", "draft"] if include_drafts else ["published"]
cap = max(1, min(int(limit), 1000))
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
seed_page = source_page if traversal_direction == "outgoing" else target_page
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
seed_conditions = or_(
*(
and_(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
seed_page.slug.in_(slugs),
)
for mapping_id, slugs in seeds.items()
)
)
stmt = (
select(LlmWikiWikiEdgeRow, source_page, target_page)
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
.where(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
source_page.is_remote_deleted.is_(False),
target_page.is_remote_deleted.is_(False),
source_page.status.in_(statuses),
target_page.status.in_(statuses),
seed_conditions,
)
.order_by(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
seed_page.slug,
neighbor_page.slug,
)
.limit(cap)
)
async with self._sf() as session:
triples = list((await session.execute(stmt)).all())
rows: list[dict[str, Any]] = []
for edge, source, target in triples:
seed = source if traversal_direction == "outgoing" else target
neighbor = target if traversal_direction == "outgoing" else source
rows.append(
{
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
"seed_slug": seed.slug,
"neighbor_slug": neighbor.slug,
"direction": traversal_direction,
"edge_type": edge.edge_type,
"source_slug": source.slug,
"target_slug": target.slug,
"page": _dict(neighbor),
}
)
return rows
result: list[dict[str, Any]] = []
if direction in {"outgoing", "both"}:
result.extend(await _query("outgoing"))
if direction in {"incoming", "both"}:
result.extend(await _query("incoming"))
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
deduplicated: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for row in result:
key = (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
if key in seen:
continue
seen.add(key)
deduplicated.append(row)
if len(deduplicated) >= cap:
break
return deduplicated
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
async with self._sf() as session:
if owner is not None:
if not mapping_id or not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
conditions = [LlmWikiWikiPageRow.id == page_id]
if mapping_id is not None:
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
result = await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.where(*conditions)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
)
.with_for_update()
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
if row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
getattr(row, key) != value for key, value in page.items()
)
for key, value in page.items():
setattr(row, key, value)
now = _now()
row.last_seen_sync_id = sync_id
row.last_error = None
row.is_remote_deleted = False
row.updated_at = now
if durable_changed:
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"changed": durable_changed, "page": _dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
@@ -115,10 +480,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
@@ -136,9 +515,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
@@ -287,17 +680,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
return False
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
@@ -329,12 +741,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if result is not None:
state.last_sync_result = dict(result)
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)