feat: add local LLMWiki wiki vector index

This commit is contained in:
Administrator committed 2026-08-24 00:49:31 +08:00
1 parent 4c9f75f31e
commit 3785345353
48 files changed
+3650 -152

No files matched your search

@@ -0,0 +1,143 @@
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
Revision ID: 20260823_01
Revises: 20260821_02
Create Date: 2026-08-23
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mysql
revision = "20260823_01"
down_revision = "20260821_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
if "wiki_index_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
if "external_search_enabled" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
if "external_search_updated_at" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
if not inspector.has_table("llmwiki_wiki_pages"):
op.create_table(
"llmwiki_wiki_pages",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
sa.Column("remote_page_id", sa.String(128), nullable=True),
sa.Column("slug", sa.String(1024), nullable=False),
sa.Column("slug_hash", sa.String(64), nullable=False),
sa.Column("title", sa.String(512), nullable=False, server_default=""),
sa.Column("summary", long_text, nullable=False),
sa.Column("content_md", long_text, nullable=False),
sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
sa.Column("status", sa.String(32), nullable=False, server_default="published"),
sa.Column("aliases", portable_json, nullable=False),
sa.Column("category_path", portable_json, nullable=False),
sa.Column("parent_slug", sa.String(1024), nullable=True),
sa.Column("wiki_path", sa.String(1024), nullable=True),
sa.Column("source_refs", portable_json, nullable=False),
sa.Column("chunk_refs", portable_json, nullable=False),
sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
)
op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
if not inspector.has_table("llmwiki_wiki_vectors"):
op.create_table(
"llmwiki_wiki_vectors",
sa.Column("id", sa.String(36), primary_key=True),
sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
sa.Column("section_index", sa.Integer(), nullable=False),
sa.Column("heading", sa.String(1024), nullable=True),
sa.Column("section_content", long_text, nullable=False),
sa.Column("content_hash", sa.String(64), nullable=False),
sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
)
op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
if not inspector.has_table("llmwiki_wiki_sync_states"):
op.create_table(
"llmwiki_wiki_sync_states",
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
sa.Column("active_sync_id", sa.String(36), nullable=True),
sa.Column("lease_owner", sa.String(128), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
if inspector.has_table(table):
op.drop_table(table)
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
if index in indexes:
op.drop_index(index, table_name="llmwiki_knowledge_bases")
for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
if column in columns:
op.drop_column("llmwiki_knowledge_bases", column)