feat: add local LLMWiki wiki vector index
This commit is contained in:
1 parent
4c9f75f31e
commit
3785345353
48 files changed
+3650
-152
No files matched your search
+143
@@ -0,0 +1,143 @@
|
||||
"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
|
||||
|
||||
Revision ID: 20260823_01
|
||||
Revises: 20260821_02
|
||||
Create Date: 2026-08-23
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = "20260823_01"
|
||||
down_revision = "20260821_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
if "wiki_index_enabled" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
|
||||
if "external_search_enabled" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
|
||||
if "external_search_updated_at" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
|
||||
mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
|
||||
if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
|
||||
op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
|
||||
if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
|
||||
op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
|
||||
|
||||
long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
|
||||
portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
|
||||
if not inspector.has_table("llmwiki_wiki_pages"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_pages",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("remote_page_id", sa.String(128), nullable=True),
|
||||
sa.Column("slug", sa.String(1024), nullable=False),
|
||||
sa.Column("slug_hash", sa.String(64), nullable=False),
|
||||
sa.Column("title", sa.String(512), nullable=False, server_default=""),
|
||||
sa.Column("summary", long_text, nullable=False),
|
||||
sa.Column("content_md", long_text, nullable=False),
|
||||
sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
|
||||
sa.Column("status", sa.String(32), nullable=False, server_default="published"),
|
||||
sa.Column("aliases", portable_json, nullable=False),
|
||||
sa.Column("category_path", portable_json, nullable=False),
|
||||
sa.Column("parent_slug", sa.String(1024), nullable=True),
|
||||
sa.Column("wiki_path", sa.String(1024), nullable=True),
|
||||
sa.Column("source_refs", portable_json, nullable=False),
|
||||
sa.Column("chunk_refs", portable_json, nullable=False),
|
||||
sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False),
|
||||
sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
|
||||
sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
|
||||
sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
|
||||
)
|
||||
op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
|
||||
op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
|
||||
op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
|
||||
|
||||
if not inspector.has_table("llmwiki_wiki_vectors"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_vectors",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
|
||||
sa.Column("section_index", sa.Integer(), nullable=False),
|
||||
sa.Column("heading", sa.String(1024), nullable=True),
|
||||
sa.Column("section_content", long_text, nullable=False),
|
||||
sa.Column("content_hash", sa.String(64), nullable=False),
|
||||
sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
|
||||
sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
|
||||
)
|
||||
op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
|
||||
op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
|
||||
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
op.create_table(
|
||||
"llmwiki_wiki_sync_states",
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
|
||||
sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
|
||||
sa.Column("active_sync_id", sa.String(36), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(128), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
|
||||
if inspector.has_table(table):
|
||||
op.drop_table(table)
|
||||
columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
|
||||
for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
|
||||
if index in indexes:
|
||||
op.drop_index(index, table_name="llmwiki_knowledge_bases")
|
||||
for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
|
||||
if column in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", column)
|
||||
Reference in new issue
Block a user