sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
+8
-6
@@ -2,16 +2,17 @@
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -19,6 +20,7 @@ __all__ = [
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiEdgeRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
|
||||
+113
-5
@@ -14,8 +14,85 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return local Wiki body pages containing any token.
|
||||
|
||||
The default keeps lightweight test doubles and third-party stores
|
||||
backward compatible. Database and in-memory stores override it.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Replace one mapping's directed Wiki graph snapshot.
|
||||
|
||||
The default keeps lightweight third-party/test stores compatible. An
|
||||
implementation must resolve both endpoints inside ``mapping_id`` and
|
||||
discard dangling or cross-mapping edges. ``None`` means the caller no
|
||||
longer owns the supplied sync lease and no mutation was made.
|
||||
"""
|
||||
|
||||
return {"edge_count": 0, "changed": False}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return one-hop neighbors within the explicitly allowed mappings.
|
||||
|
||||
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
|
||||
retains the exact seed provenance. A flat slug list is applied only to
|
||||
the supplied ``mapping_ids`` and never broadens authorization.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Refresh a mirrored page without replacing its vectors.
|
||||
|
||||
The write is fenced by the mapping sync lease. ``None`` means the
|
||||
caller no longer owns that lease (or the expected page disappeared),
|
||||
while the returned ``changed`` flag reports whether durable page
|
||||
metadata changed. The page is always marked as seen by ``sync_id``.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
@@ -25,13 +102,34 @@ class LlmWikiIndexStore(ABC):
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
"""Retire pages absent from a fenced full scan.
|
||||
|
||||
``None`` means the supplied owner/sync pair no longer owns the mapping
|
||||
lease and no mutation was made.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
@@ -73,7 +171,17 @@ class LlmWikiIndexStore(ABC):
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
+275
-18
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.edges: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
def _has_current_sync_lease(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
state = self.states.get(mapping_id)
|
||||
return bool(
|
||||
state is not None
|
||||
and state.get("lease_owner") == owner
|
||||
and state.get("active_sync_id") == sync_id
|
||||
and isinstance(state.get("lease_expires_at"), datetime)
|
||||
and state["lease_expires_at"] >= _now()
|
||||
)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
@@ -30,12 +47,219 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids)
|
||||
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
|
||||
rows = []
|
||||
for page in self.pages.values():
|
||||
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
|
||||
if needles and any(needle in haystack for needle in needles):
|
||||
rows.append(dict(page))
|
||||
if len(rows) >= max(1, limit):
|
||||
break
|
||||
return rows
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
replacement = [
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source]["id"],
|
||||
"target_page_id": pages_by_slug[target]["id"],
|
||||
"source_slug": source,
|
||||
"target_slug": target,
|
||||
"edge_type": edge_type,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
current = self.edges.get(mapping_id, [])
|
||||
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
|
||||
changed = current_keys != canonical
|
||||
if changed:
|
||||
self.edges[mapping_id] = replacement
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"edge_count": len(replacement), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
|
||||
else:
|
||||
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
|
||||
result: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for mapping_id in sorted(allowed):
|
||||
mapping_seeds = seeds.get(mapping_id) or set()
|
||||
for edge in self.edges.get(mapping_id, []):
|
||||
traversals: list[tuple[str, str, str, str]] = []
|
||||
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["source_slug"],
|
||||
edge["target_slug"],
|
||||
"outgoing",
|
||||
edge["target_page_id"],
|
||||
)
|
||||
)
|
||||
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["target_slug"],
|
||||
edge["source_slug"],
|
||||
"incoming",
|
||||
edge["source_page_id"],
|
||||
)
|
||||
)
|
||||
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
|
||||
page = pages_by_id.get(page_id)
|
||||
if not page or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"seed_slug": seed_slug,
|
||||
"neighbor_slug": neighbor_slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge["edge_type"],
|
||||
"source_slug": edge["source_slug"],
|
||||
"target_slug": edge["target_slug"],
|
||||
"page": dict(page),
|
||||
}
|
||||
)
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
return result[: max(1, min(int(limit), 1000))]
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
page = self.pages.get(page_id)
|
||||
if page is None:
|
||||
return False
|
||||
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
|
||||
if mapping_id is not None and mapping_id != actual_mapping_id:
|
||||
return False
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
actual_mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
page["last_seen_sync_id"] = sync_id
|
||||
page["last_error"] = None
|
||||
return True
|
||||
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
|
||||
if existing is None:
|
||||
return None
|
||||
durable_changed = any(existing.get(key) != value for key, value in page.items())
|
||||
row = self.pages[str(existing["id"])]
|
||||
row.update(page)
|
||||
row.update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"updated_at": _now().isoformat(),
|
||||
}
|
||||
)
|
||||
if durable_changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"changed": durable_changed, "page": dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
@@ -59,7 +283,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
@@ -70,11 +308,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
return True
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
@@ -82,6 +329,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
|
||||
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
@@ -131,17 +380,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
@@ -158,8 +417,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
@@ -168,11 +429,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
|
||||
+48
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiEdgeRow(Base):
|
||||
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
|
||||
|
||||
__tablename__ = "llmwiki_wiki_edges"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"knowledge_base_mapping_id",
|
||||
"target_page_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
source_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
target_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+396
-10
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def _fence_current_sync_lease(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
"""Fence destructive snapshot writes behind the current live lease."""
|
||||
|
||||
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
|
||||
# ignores FOR UPDATE, while this statement obtains its write lock on
|
||||
# every supported backend before the page/edge mutation starts.
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(updated_at=_now())
|
||||
)
|
||||
return bool((await session.execute(stmt)).rowcount)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
@@ -53,18 +85,322 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
|
||||
if not normalized_ids or not normalized_tokens:
|
||||
return []
|
||||
conditions = []
|
||||
for token in normalized_tokens[:32]:
|
||||
needle = f"%{token}%"
|
||||
conditions.extend(
|
||||
(
|
||||
func.lower(LlmWikiWikiPageRow.title).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.summary).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
|
||||
)
|
||||
)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(statuses),
|
||||
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
|
||||
or_(*conditions),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.updated_at.desc())
|
||||
.limit(max(1, min(int(limit), 2000)))
|
||||
)
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomically replace an authorized mapping's resolved edge snapshot."""
|
||||
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
page_stmt = select(LlmWikiWikiPageRow).where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
pages = list((await session.execute(page_stmt)).scalars())
|
||||
pages_by_slug = {page.slug: page for page in pages}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
existing_stmt = select(
|
||||
LlmWikiWikiEdgeRow.source_page_id,
|
||||
LlmWikiWikiEdgeRow.target_page_id,
|
||||
LlmWikiWikiEdgeRow.edge_type,
|
||||
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
|
||||
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
|
||||
replacement_ids = {
|
||||
(
|
||||
pages_by_slug[source].id,
|
||||
pages_by_slug[target].id,
|
||||
edge_type,
|
||||
)
|
||||
for source, target, edge_type in canonical
|
||||
}
|
||||
changed = existing != replacement_ids
|
||||
if changed:
|
||||
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
|
||||
now = _now()
|
||||
edge_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source].id,
|
||||
"target_page_id": pages_by_slug[target].id,
|
||||
"edge_type": edge_type,
|
||||
"created_at": now,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
for start in range(0, len(edge_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiEdgeRow),
|
||||
edge_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"edge_count": len(canonical), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
if not allowed:
|
||||
return []
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
|
||||
else:
|
||||
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
|
||||
if not seeds:
|
||||
return []
|
||||
|
||||
source_page = aliased(LlmWikiWikiPageRow)
|
||||
target_page = aliased(LlmWikiWikiPageRow)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
cap = max(1, min(int(limit), 1000))
|
||||
|
||||
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
|
||||
seed_page = source_page if traversal_direction == "outgoing" else target_page
|
||||
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
|
||||
seed_conditions = or_(
|
||||
*(
|
||||
and_(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
|
||||
seed_page.slug.in_(slugs),
|
||||
)
|
||||
for mapping_id, slugs in seeds.items()
|
||||
)
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiEdgeRow, source_page, target_page)
|
||||
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
|
||||
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
|
||||
.where(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
|
||||
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
source_page.is_remote_deleted.is_(False),
|
||||
target_page.is_remote_deleted.is_(False),
|
||||
source_page.status.in_(statuses),
|
||||
target_page.status.in_(statuses),
|
||||
seed_conditions,
|
||||
)
|
||||
.order_by(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
seed_page.slug,
|
||||
neighbor_page.slug,
|
||||
)
|
||||
.limit(cap)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
triples = list((await session.execute(stmt)).all())
|
||||
rows: list[dict[str, Any]] = []
|
||||
for edge, source, target in triples:
|
||||
seed = source if traversal_direction == "outgoing" else target
|
||||
neighbor = target if traversal_direction == "outgoing" else source
|
||||
rows.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
|
||||
"seed_slug": seed.slug,
|
||||
"neighbor_slug": neighbor.slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge.edge_type,
|
||||
"source_slug": source.slug,
|
||||
"target_slug": target.slug,
|
||||
"page": _dict(neighbor),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
if direction in {"outgoing", "both"}:
|
||||
result.extend(await _query("outgoing"))
|
||||
if direction in {"incoming", "both"}:
|
||||
result.extend(await _query("incoming"))
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
deduplicated: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for row in result:
|
||||
key = (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduplicated.append(row)
|
||||
if len(deduplicated) >= cap:
|
||||
break
|
||||
return deduplicated
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if owner is not None:
|
||||
if not mapping_id or not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
conditions = [LlmWikiWikiPageRow.id == page_id]
|
||||
if mapping_id is not None:
|
||||
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.where(*conditions)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
|
||||
getattr(row, key) != value for key, value in page.items()
|
||||
)
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
now = _now()
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.updated_at = now
|
||||
if durable_changed:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"changed": durable_changed, "page": _dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
@@ -115,10 +451,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
@@ -136,9 +486,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
@@ -287,17 +651,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
return False
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
@@ -329,12 +712,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if result is not None:
|
||||
state.last_sync_result = dict(result)
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
|
||||
Reference in new issue
Block a user