sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -2,16 +2,17 @@
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
__all__ = [
@@ -19,6 +20,7 @@ __all__ = [
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiEdgeRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
@@ -14,8 +14,85 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return local Wiki body pages containing any token.
The default keeps lightweight test doubles and third-party stores
backward compatible. Database and in-memory stores override it.
"""
return []
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Replace one mapping's directed Wiki graph snapshot.
The default keeps lightweight third-party/test stores compatible. An
implementation must resolve both endpoints inside ``mapping_id`` and
discard dangling or cross-mapping edges. ``None`` means the caller no
longer owns the supplied sync lease and no mutation was made.
"""
return {"edge_count": 0, "changed": False}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return one-hop neighbors within the explicitly allowed mappings.
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
retains the exact seed provenance. A flat slug list is applied only to
the supplied ``mapping_ids`` and never broadens authorization.
"""
return []
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
"""Refresh a mirrored page without replacing its vectors.
The write is fenced by the mapping sync lease. ``None`` means the
caller no longer owns that lease (or the expected page disappeared),
while the returned ``changed`` flag reports whether durable page
metadata changed. The page is always marked as seen by ``sync_id``.
"""
...
@abstractmethod
async def replace_page_vectors(
@@ -25,13 +102,34 @@ class LlmWikiIndexStore(ABC):
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
owner: str | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
"""Retire pages absent from a fenced full scan.
``None`` means the supplied owner/sync pair no longer owns the mapping
lease and no mutation was made.
"""
...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
@@ -73,7 +171,17 @@ class LlmWikiIndexStore(ABC):
) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.edges: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
def _has_current_sync_lease(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
state = self.states.get(mapping_id)
return bool(
state is not None
and state.get("lease_owner") == owner
and state.get("active_sync_id") == sync_id
and isinstance(state.get("lease_expires_at"), datetime)
and state["lease_expires_at"] >= _now()
)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
@@ -30,12 +47,219 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
wanted = set(mapping_ids)
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
rows = []
for page in self.pages.values():
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
if needles and any(needle in haystack for needle in needles):
rows.append(dict(page))
if len(rows) >= max(1, limit):
break
return rows
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
replacement = [
{
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source]["id"],
"target_page_id": pages_by_slug[target]["id"],
"source_slug": source,
"target_slug": target,
"edge_type": edge_type,
}
for source, target, edge_type in sorted(canonical)
]
current = self.edges.get(mapping_id, [])
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
changed = current_keys != canonical
if changed:
self.edges[mapping_id] = replacement
self._state(mapping_id)["index_revision"] += 1
return {"edge_count": len(replacement), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
else:
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
seeds = {mapping_id: shared for mapping_id in allowed}
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
result: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for mapping_id in sorted(allowed):
mapping_seeds = seeds.get(mapping_id) or set()
for edge in self.edges.get(mapping_id, []):
traversals: list[tuple[str, str, str, str]] = []
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
traversals.append(
(
edge["source_slug"],
edge["target_slug"],
"outgoing",
edge["target_page_id"],
)
)
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
traversals.append(
(
edge["target_slug"],
edge["source_slug"],
"incoming",
edge["source_page_id"],
)
)
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
page = pages_by_id.get(page_id)
if not page or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
if key in seen:
continue
seen.add(key)
result.append(
{
"knowledge_base_mapping_id": mapping_id,
"seed_slug": seed_slug,
"neighbor_slug": neighbor_slug,
"direction": traversal_direction,
"edge_type": edge["edge_type"],
"source_slug": edge["source_slug"],
"target_slug": edge["target_slug"],
"page": dict(page),
}
)
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
return result[: max(1, min(int(limit), 1000))]
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
page = self.pages.get(page_id)
if page is None:
return False
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
if mapping_id is not None and mapping_id != actual_mapping_id:
return False
if owner is not None and not self._has_current_sync_lease(
actual_mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
page["last_seen_sync_id"] = sync_id
page["last_error"] = None
return True
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
if existing is None:
return None
durable_changed = any(existing.get(key) != value for key, value in page.items())
row = self.pages[str(existing["id"])]
row.update(page)
row.update(
{
"last_seen_sync_id": sync_id,
"last_error": None,
"is_remote_deleted": False,
"updated_at": _now().isoformat(),
}
)
if durable_changed:
self._state(mapping_id)["index_revision"] += 1
return {"changed": durable_changed, "page": dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
@@ -59,7 +283,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
@@ -70,11 +308,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
return True
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
@@ -82,6 +329,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
self.vectors.pop(page["id"], None)
changed += 1
if changed:
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
self._state(mapping_id)["index_revision"] += 1
return changed
@@ -131,17 +380,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
sync_id: str,
remote_page_count: int,
) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
state["remote_page_count"] = max(0, remote_page_count)
state["updated_at"] = _now()
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
@@ -158,8 +417,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
}
)
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
@@ -168,11 +429,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
result = []
for page in sorted(
(
page
for page in self.pages.values()
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
),
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
):
result.append(
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiEdgeRow(Base):
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
__tablename__ = "llmwiki_wiki_edges"
__table_args__ = (
UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
Index(
"ix_llmwiki_wiki_edge_mapping_source",
"knowledge_base_mapping_id",
"source_page_id",
),
Index(
"ix_llmwiki_wiki_edge_mapping_target",
"knowledge_base_mapping_id",
"target_page_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
source_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
target_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import aliased
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
def _now() -> datetime:
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.flush()
return row
async def _fence_current_sync_lease(
self,
session: AsyncSession,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
"""Fence destructive snapshot writes behind the current live lease."""
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
# ignores FOR UPDATE, while this statement obtains its write lock on
# every supported backend before the page/edge mutation starts.
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(updated_at=_now())
)
return bool((await session.execute(stmt)).rowcount)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
@@ -53,18 +85,322 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
if not normalized_ids or not normalized_tokens:
return []
conditions = []
for token in normalized_tokens[:32]:
needle = f"%{token}%"
conditions.extend(
(
func.lower(LlmWikiWikiPageRow.title).like(needle),
func.lower(LlmWikiWikiPageRow.summary).like(needle),
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
)
)
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
or_(*conditions),
)
.order_by(LlmWikiWikiPageRow.updated_at.desc())
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
await session.execute(
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Atomically replace an authorized mapping's resolved edge snapshot."""
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
page_stmt = select(LlmWikiWikiPageRow).where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
pages = list((await session.execute(page_stmt)).scalars())
pages_by_slug = {page.slug: page for page in pages}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
existing_stmt = select(
LlmWikiWikiEdgeRow.source_page_id,
LlmWikiWikiEdgeRow.target_page_id,
LlmWikiWikiEdgeRow.edge_type,
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
replacement_ids = {
(
pages_by_slug[source].id,
pages_by_slug[target].id,
edge_type,
)
for source, target, edge_type in canonical
}
changed = existing != replacement_ids
if changed:
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
now = _now()
edge_rows = [
{
"id": str(uuid4()),
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source].id,
"target_page_id": pages_by_slug[target].id,
"edge_type": edge_type,
"created_at": now,
}
for source, target, edge_type in sorted(canonical)
]
for start in range(0, len(edge_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiEdgeRow),
edge_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"edge_count": len(canonical), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
if not allowed:
return []
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
else:
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
seeds = {mapping_id: shared for mapping_id in allowed}
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
if not seeds:
return []
source_page = aliased(LlmWikiWikiPageRow)
target_page = aliased(LlmWikiWikiPageRow)
statuses = ["published", "draft"] if include_drafts else ["published"]
cap = max(1, min(int(limit), 1000))
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
seed_page = source_page if traversal_direction == "outgoing" else target_page
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
seed_conditions = or_(
*(
and_(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
seed_page.slug.in_(slugs),
)
for mapping_id, slugs in seeds.items()
)
)
stmt = (
select(LlmWikiWikiEdgeRow, source_page, target_page)
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
.where(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
source_page.is_remote_deleted.is_(False),
target_page.is_remote_deleted.is_(False),
source_page.status.in_(statuses),
target_page.status.in_(statuses),
seed_conditions,
)
.order_by(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
seed_page.slug,
neighbor_page.slug,
)
.limit(cap)
)
async with self._sf() as session:
triples = list((await session.execute(stmt)).all())
rows: list[dict[str, Any]] = []
for edge, source, target in triples:
seed = source if traversal_direction == "outgoing" else target
neighbor = target if traversal_direction == "outgoing" else source
rows.append(
{
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
"seed_slug": seed.slug,
"neighbor_slug": neighbor.slug,
"direction": traversal_direction,
"edge_type": edge.edge_type,
"source_slug": source.slug,
"target_slug": target.slug,
"page": _dict(neighbor),
}
)
return rows
result: list[dict[str, Any]] = []
if direction in {"outgoing", "both"}:
result.extend(await _query("outgoing"))
if direction in {"incoming", "both"}:
result.extend(await _query("incoming"))
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
deduplicated: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for row in result:
key = (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
if key in seen:
continue
seen.add(key)
deduplicated.append(row)
if len(deduplicated) >= cap:
break
return deduplicated
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
async with self._sf() as session:
if owner is not None:
if not mapping_id or not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
conditions = [LlmWikiWikiPageRow.id == page_id]
if mapping_id is not None:
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
result = await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.where(*conditions)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
)
.with_for_update()
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
if row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
getattr(row, key) != value for key, value in page.items()
)
for key, value in page.items():
setattr(row, key, value)
now = _now()
row.last_seen_sync_id = sync_id
row.last_error = None
row.is_remote_deleted = False
row.updated_at = now
if durable_changed:
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"changed": durable_changed, "page": _dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
@@ -115,10 +451,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
@@ -136,9 +486,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
@@ -287,17 +651,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
return False
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
@@ -329,12 +712,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if result is not None:
state.last_sync_result = dict(result)
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)