初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence facade for the DeerFlow-local Wiki vector index."""
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LlmWikiIndexStore",
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
]
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Iterate vector rows for streaming export.
|
||||
|
||||
Implementations with a database backend should override this method so
|
||||
very large vector packages do not need to be materialized in memory.
|
||||
"""
|
||||
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
|
||||
for row in snapshot.get("rows") or []:
|
||||
if isinstance(row, dict):
|
||||
yield row
|
||||
|
||||
@abstractmethod
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
"""In-memory local Wiki index store used by tests and DB-less mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
row = {
|
||||
**(existing or {}),
|
||||
**page,
|
||||
"id": page_id,
|
||||
"index_status": "ready",
|
||||
"indexed_fingerprint": fingerprint,
|
||||
"indexed_at": now,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"last_seen_sync_id": sync_id,
|
||||
"created_at": (existing or {}).get("created_at", now),
|
||||
"updated_at": now,
|
||||
}
|
||||
self.pages[page_id] = row
|
||||
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
|
||||
state = self._state(page["knowledge_base_mapping_id"])
|
||||
state["index_revision"] += 1
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": error,
|
||||
# Preserve the last atomically completed generation.
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
page.update({"is_remote_deleted": True, "index_status": "deleted"})
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
rows = []
|
||||
for page_id, vectors in self.vectors.items():
|
||||
page = self.pages[page_id]
|
||||
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
|
||||
continue
|
||||
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
|
||||
state = self._state(mapping_id)
|
||||
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
state = self._state(mapping_id)
|
||||
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
now = _now()
|
||||
expires = state.get("lease_expires_at")
|
||||
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
|
||||
return False
|
||||
state.update(
|
||||
{
|
||||
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
|
||||
"active_sync_id": sync_id,
|
||||
"lease_owner": owner,
|
||||
"lease_expires_at": now + timedelta(seconds=lease_seconds),
|
||||
"last_started_at": now,
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner:
|
||||
return False
|
||||
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
|
||||
return True
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return False
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
{
|
||||
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
|
||||
"lease_owner": None,
|
||||
"lease_expires_at": None,
|
||||
"active_sync_id": None,
|
||||
"last_error": error,
|
||||
"remote_page_count": remote_page_count,
|
||||
"local_page_count": len(pages),
|
||||
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
|
||||
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
}
|
||||
)
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": len(self.vectors.get(page["id"], [])),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
for mapping_id in mapping_ids:
|
||||
state = self._state(mapping_id)
|
||||
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
|
||||
page["index_status"] = "stale"
|
||||
return len(mapping_ids)
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class LlmWikiWikiPageRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_pages"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
|
||||
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
|
||||
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
|
||||
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
|
||||
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
|
||||
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
|
||||
|
||||
|
||||
class LlmWikiWikiVectorRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_vectors"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
|
||||
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
|
||||
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
|
||||
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
"""SQLAlchemy implementation of the local Wiki index store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _dict(row) -> dict[str, Any]:
|
||||
value = row.to_dict()
|
||||
for key, item in tuple(value.items()):
|
||||
if isinstance(item, datetime):
|
||||
value[key] = item.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
|
||||
self._sf = session_factory
|
||||
self.write_batch_size = max(1, min(int(write_batch_size), 500))
|
||||
|
||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if row is None:
|
||||
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
async with self._sf() as session:
|
||||
result = (await session.execute(stmt)).one_or_none()
|
||||
return {**_dict(result[0]), "vector_count": result[1]} if result else None
|
||||
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
else:
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
row.index_status = "ready"
|
||||
row.indexed_fingerprint = fingerprint
|
||||
row.indexed_at = now
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.updated_at = now
|
||||
await session.flush()
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||
vector_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"page_id": row.id,
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"section_index": int(item["section_index"]),
|
||||
"heading": item.get("heading"),
|
||||
"section_content": str(item["section_content"]),
|
||||
"content_hash": str(item["content_hash"]),
|
||||
"embedding_fingerprint": fingerprint,
|
||||
"embedding_dimensions": int(item["embedding_dimensions"]),
|
||||
"vector_blob": bytes(item["vector_blob"]),
|
||||
"created_at": now,
|
||||
}
|
||||
for item in vectors
|
||||
]
|
||||
# Keep the page replacement atomic, but cap each SQL statement so
|
||||
# large Wiki pages cannot flood the database with one huge insert.
|
||||
for start in range(0, len(vector_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiVectorRow),
|
||||
vector_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
if row is None:
|
||||
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
|
||||
session.add(row)
|
||||
row.index_status = "failed"
|
||||
else:
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
|
||||
# Preserve the last atomically completed generation while the
|
||||
# newer remote version remains pending after an embedding error.
|
||||
row.index_status = "ready" if vector_count else "failed"
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = safe_error
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
now = _now()
|
||||
page_ids = [row.id for row in rows]
|
||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
|
||||
for row in rows:
|
||||
row.is_remote_deleted = True
|
||||
row.index_status = "deleted"
|
||||
row.updated_at = now
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
|
||||
|
||||
async def iter_vector_snapshot_rows(
|
||||
self,
|
||||
mapping_id: str,
|
||||
fingerprint: str,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
batch = max(1, min(int(batch_size), 5000))
|
||||
offset = 0
|
||||
while True:
|
||||
stmt = (
|
||||
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
|
||||
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
|
||||
.where(
|
||||
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
|
||||
LlmWikiWikiPageRow.index_status == "ready",
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
|
||||
.limit(batch)
|
||||
.offset(offset)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
pairs = list((await session.execute(stmt)).all())
|
||||
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield row
|
||||
if len(rows) < batch:
|
||||
break
|
||||
offset += len(rows)
|
||||
|
||||
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None:
|
||||
return 0, None, "not_synced"
|
||||
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
|
||||
|
||||
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
try:
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another worker created the one-per-KB state row between our
|
||||
# read and insert. Roll back this isolated bootstrap session;
|
||||
# the conditional UPDATE below is the actual lease election.
|
||||
await session.rollback()
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
or_(
|
||||
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at < now,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
|
||||
active_sync_id=sync_id,
|
||||
lease_owner=owner,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_started_at=now,
|
||||
last_error=None,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
|
||||
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def update_sync_progress(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async with self._sf() as session:
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
failed_count = int(
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
*visible,
|
||||
or_(
|
||||
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
|
||||
LlmWikiWikiPageRow.last_error.is_not(None),
|
||||
),
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
)
|
||||
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
|
||||
now = _now()
|
||||
was_rebuilding = state.state == "rebuilding"
|
||||
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
|
||||
state.active_sync_id = None
|
||||
state.lease_owner = None
|
||||
state.lease_expires_at = None
|
||||
state.last_error = (error or "")[:4000] or None
|
||||
state.remote_page_count = remote_page_count
|
||||
state.local_page_count = local_count
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
if mapping_ids:
|
||||
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
|
||||
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
vector_counts = (
|
||||
select(
|
||||
LlmWikiWikiVectorRow.page_id.label("page_id"),
|
||||
func.count(LlmWikiWikiVectorRow.id).label("vector_count"),
|
||||
)
|
||||
.where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id)
|
||||
.group_by(LlmWikiWikiVectorRow.page_id)
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow, func.coalesce(vector_counts.c.vector_count, 0))
|
||||
.outerjoin(vector_counts, vector_counts.c.page_id == LlmWikiWikiPageRow.id)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.wiki_path, LlmWikiWikiPageRow.slug)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row, vector_count in rows:
|
||||
page = _dict(row)
|
||||
result.append(
|
||||
{
|
||||
"id": page["id"],
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"page_type": page["page_type"],
|
||||
"status": page["status"],
|
||||
"index_status": page["index_status"],
|
||||
"vector_count": int(vector_count or 0),
|
||||
"indexed_at": page.get("indexed_at"),
|
||||
"last_error": page.get("last_error"),
|
||||
"last_seen_sync_id": page.get("last_seen_sync_id"),
|
||||
"updated_at": page.get("updated_at"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
|
||||
if not mapping_ids:
|
||||
return 0
|
||||
now = _now()
|
||||
async with self._sf() as session:
|
||||
for mapping_id in mapping_ids:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.state = "rebuild_required"
|
||||
state.embedding_fingerprint = fingerprint
|
||||
state.updated_at = now
|
||||
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
|
||||
await session.commit()
|
||||
return len(mapping_ids)
|
||||
Reference in new issue
Block a user