feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -22,7 +22,15 @@ class LlmWikiStore(ABC):
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def adopt_conversation_deposit_mapping(
@@ -33,6 +33,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
"published_by": None,
"reviewed_by": None,
"published_at": None,
"wiki_index_enabled": True,
"external_search_enabled": False,
"external_search_updated_at": None,
"created_at": now,
"updated_at": now,
}
@@ -65,7 +68,15 @@ class MemoryLlmWikiStore(LlmWikiStore):
return dict(row)
return None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
@@ -73,6 +84,11 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
row["updated_at"] = _now()
return dict(row)
@@ -143,11 +159,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if (
row.get("owner_user_id") == "system"
and row.get("name") == "对话沉淀"
and row.get("publication_status") == "published"
):
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, UniqueConstraint
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,9 @@ class LlmWikiKnowledgeBaseRow(Base):
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
@@ -19,7 +19,7 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("published_at", "created_at", "updated_at"):
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
@@ -82,7 +82,15 @@ class LlmWikiRepository(LlmWikiStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
async def update_mapping(
self,
mapping_id: str,
*,
name: str | None = None,
description: str | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
@@ -91,6 +99,11 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
@@ -0,0 +1,21 @@
"""Persistence facade for the DeerFlow-local Wiki vector index."""
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory):
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
__all__ = [
"LlmWikiIndexStore",
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
]
@@ -0,0 +1,54 @@
"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
@abstractmethod
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@abstractmethod
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
@abstractmethod
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@abstractmethod
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
@@ -0,0 +1,160 @@
"""In-memory local Wiki index store used by tests and DB-less mode."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
def _now() -> datetime:
return datetime.now(UTC)
class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return dict(row) if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
row = {
**(existing or {}),
**page,
"id": page_id,
"index_status": "ready",
"indexed_fingerprint": fingerprint,
"indexed_at": now,
"last_error": None,
"is_remote_deleted": False,
"last_seen_sync_id": sync_id,
"created_at": (existing or {}).get("created_at", now),
"updated_at": now,
}
self.pages[page_id] = row
self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
state = self._state(page["knowledge_base_mapping_id"])
state["index_revision"] += 1
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
{
"last_seen_sync_id": sync_id,
"last_error": error,
# Preserve the last atomically completed generation.
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
page.update({"is_remote_deleted": True, "index_status": "deleted"})
self.vectors.pop(page["id"], None)
changed += 1
if changed:
self._state(mapping_id)["index_revision"] += 1
return changed
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
rows = []
for page_id, vectors in self.vectors.items():
page = self.pages[page_id]
if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
continue
rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
state = self._state(mapping_id)
return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
state = self._state(mapping_id)
return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
state = self._state(mapping_id)
now = _now()
expires = state.get("lease_expires_at")
if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
return False
state.update(
{
"state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
"active_sync_id": sync_id,
"lease_owner": owner,
"lease_expires_at": now + timedelta(seconds=lease_seconds),
"last_started_at": now,
}
)
return True
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner:
return False
state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
{
"state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
"lease_owner": None,
"lease_expires_at": None,
"active_sync_id": None,
"last_error": error,
"remote_page_count": remote_page_count,
"local_page_count": len(pages),
"ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
"failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
}
)
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
for mapping_id in mapping_ids:
state = self._state(mapping_id)
state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
page["index_status"] = "stale"
return len(mapping_ids)
@@ -0,0 +1,94 @@
"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class LlmWikiWikiPageRow(Base):
__tablename__ = "llmwiki_wiki_pages"
__table_args__ = (
UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
slug: Mapped[str] = mapped_column(String(1024), nullable=False)
slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
class LlmWikiWikiVectorRow(Base):
__tablename__ = "llmwiki_wiki_vectors"
__table_args__ = (
UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -0,0 +1,295 @@
"""SQLAlchemy implementation of the local Wiki index store."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row) -> dict[str, Any]:
value = row.to_dict()
for key, item in tuple(value.items()):
if isinstance(item, datetime):
value[key] = item.isoformat()
return value
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if row is None:
row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
session.add(row)
await session.flush()
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async with self._sf() as session:
await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
else:
for key, value in page.items():
setattr(row, key, value)
row.index_status = "ready"
row.indexed_fingerprint = fingerprint
row.indexed_at = now
row.last_error = None
row.is_remote_deleted = False
row.last_seen_sync_id = sync_id
row.updated_at = now
await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
session.add_all(
[
LlmWikiWikiVectorRow(
id=str(uuid4()),
page_id=row.id,
knowledge_base_mapping_id=mapping_id,
section_index=int(item["section_index"]),
heading=item.get("heading"),
section_content=str(item["section_content"]),
content_hash=str(item["content_hash"]),
embedding_fingerprint=fingerprint,
embedding_dimensions=int(item["embedding_dimensions"]),
vector_blob=bytes(item["vector_blob"]),
created_at=now,
)
for item in vectors
]
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
if row is None:
row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
session.add(row)
row.index_status = "failed"
else:
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
# Preserve the last atomically completed generation while the
# newer remote version remains pending after an embedding error.
row.index_status = "ready" if vector_count else "failed"
row.last_seen_sync_id = sync_id
row.last_error = safe_error
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async with self._sf() as session:
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
)
.with_for_update()
)
rows = list((await session.execute(stmt)).scalars())
if not rows:
return 0
now = _now()
page_ids = [row.id for row in rows]
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
for row in rows:
row.is_remote_deleted = True
row.index_status = "deleted"
row.updated_at = now
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return len(rows)
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
)
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None:
return 0, None, "not_synced"
return int(state.index_revision or 0), state.embedding_fingerprint, state.state
async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
now = _now()
async with self._sf() as session:
try:
await self._ensure_state(session, mapping_id)
await session.commit()
except IntegrityError:
# Another worker created the one-per-KB state row between our
# read and insert. Roll back this isolated bootstrap session;
# the conditional UPDATE below is the actual lease election.
await session.rollback()
async with self._sf() as session:
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
or_(
LlmWikiWikiSyncStateRow.lease_owner.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
LlmWikiWikiSyncStateRow.lease_expires_at < now,
),
)
.values(
state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
active_sync_id=sync_id,
lease_owner=owner,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_started_at=now,
last_error=None,
updated_at=now,
)
)
result = await session.execute(stmt)
await session.commit()
return bool(result.rowcount)
async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(LlmWikiWikiSyncStateRow)
.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
.values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
failed_count = int(
(
await session.execute(
select(func.count())
.select_from(LlmWikiWikiPageRow)
.where(
*visible,
or_(
LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
LlmWikiWikiPageRow.last_error.is_not(None),
),
)
)
).scalar_one()
)
vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
now = _now()
was_rebuilding = state.state == "rebuilding"
state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
state.active_sync_id = None
state.lease_owner = None
state.lease_expires_at = None
state.last_error = (error or "")[:4000] or None
state.remote_page_count = remote_page_count
state.local_page_count = local_count
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)
if mapping_ids:
stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
if not mapping_ids:
return 0
now = _now()
async with self._sf() as session:
for mapping_id in mapping_ids:
state = await self._ensure_state(session, mapping_id)
state.state = "rebuild_required"
state.embedding_fingerprint = fingerprint
state.updated_at = now
await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
await session.commit()
return len(mapping_ids)
@@ -0,0 +1,143 @@
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
Revision ID: 20260823_01
Revises: 20260821_02
Create Date: 2026-08-23
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mysql
revision = "20260823_01"
down_revision = "20260821_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
if "wiki_index_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
if "external_search_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
if "external_search_updated_at" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
if not inspector.has_table("llmwiki_wiki_pages"):
op.create_table(
"llmwiki_wiki_pages",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
sa.Column("remote_page_id", sa.String(128), nullable=True),
sa.Column("slug", sa.String(1024), nullable=False),
sa.Column("slug_hash", sa.String(64), nullable=False),
sa.Column("title", sa.String(512), nullable=False, server_default=""),
sa.Column("summary", long_text, nullable=False),
sa.Column("content_md", long_text, nullable=False),
sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
sa.Column("status", sa.String(32), nullable=False, server_default="published"),
sa.Column("aliases", portable_json, nullable=False),
sa.Column("category_path", portable_json, nullable=False),
sa.Column("parent_slug", sa.String(1024), nullable=True),
sa.Column("wiki_path", sa.String(1024), nullable=True),
sa.Column("source_refs", portable_json, nullable=False),
sa.Column("chunk_refs", portable_json, nullable=False),
sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
)
op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
if not inspector.has_table("llmwiki_wiki_vectors"):
op.create_table(
"llmwiki_wiki_vectors",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
sa.Column("section_index", sa.Integer(), nullable=False),
sa.Column("heading", sa.String(1024), nullable=True),
sa.Column("section_content", long_text, nullable=False),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
)
op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
if not inspector.has_table("llmwiki_wiki_sync_states"):
op.create_table(
"llmwiki_wiki_sync_states",
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
sa.Column("active_sync_id", sa.String(36), nullable=True),
sa.Column("lease_owner", sa.String(128), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
if inspector.has_table(table):
op.drop_table(table)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
if index in indexes:
op.drop_index(index, table_name="llmwiki_knowledge_bases")
for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
if column in columns:
op.drop_column("llmwiki_knowledge_bases", column)
@@ -52,6 +52,7 @@ from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.models.run_event import RunEventRow
@@ -129,6 +130,9 @@ __all__ = [
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiWikiPageRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"NotificationRow",
"ParallelAgentTaskRow",