sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
+21
@@ -0,0 +1,21 @@
|
||||
"""Merge assistant-wiki metadata and report-collaboration heads.
|
||||
|
||||
Revision ID: 20260909_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
revision = "20260909_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Create durable public-document library tables.
|
||||
|
||||
Revision ID: 20260914_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260914_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("gongwen_documents"):
|
||||
op.create_table(
|
||||
"gongwen_documents",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("user_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
|
||||
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
|
||||
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
|
||||
sa.Column("template_key", sa.String(length=96), nullable=True),
|
||||
sa.Column("research_session_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("original_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("issue_counts", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
|
||||
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
|
||||
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
|
||||
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
|
||||
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
|
||||
|
||||
if not inspector.has_table("gongwen_document_versions"):
|
||||
op.create_table(
|
||||
"gongwen_document_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("document_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_no", sa.Integer(), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("file_name", sa.String(length=512), nullable=False),
|
||||
sa.Column("storage_key", sa.String(length=1024), nullable=False),
|
||||
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("review_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
)
|
||||
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
|
||||
op.create_index(
|
||||
"ix_gongwen_document_versions_document_created",
|
||||
"gongwen_document_versions",
|
||||
["document_id", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("gongwen_document_versions"):
|
||||
op.drop_table("gongwen_document_versions")
|
||||
if inspector.has_table("gongwen_documents"):
|
||||
op.drop_table("gongwen_documents")
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
"""Persist mapping-scoped links for local LLMWiki graph expansion.
|
||||
|
||||
Revision ID: 20260916_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("llmwiki_wiki_edges"):
|
||||
return
|
||||
op.create_table(
|
||||
"llmwiki_wiki_edges",
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column(
|
||||
"knowledge_base_mapping_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"source_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"target_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"edge_type",
|
||||
sa.String(length=32),
|
||||
nullable=False,
|
||||
server_default="wiki_link",
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_source_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_target_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["target_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "target_page_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
|
||||
op.drop_table("llmwiki_wiki_edges")
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
"""Merge the public-document and assistant-knowledge migration heads.
|
||||
|
||||
Revision ID: 20260916_01
|
||||
Revises: 20260909_01, 20260914_01
|
||||
Create Date: 2026-09-16
|
||||
|
||||
The feedback-scoring migration was added as ``20260916_02`` and references
|
||||
this merge revision. Keeping the merge as an explicit no-op makes the
|
||||
Alembic graph linear again without changing either existing schema branch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
revision = "20260916_01"
|
||||
down_revision = ("20260909_01", "20260914_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Record the merge; both parent revisions already own their DDL."""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Splitting the migration history requires no schema change."""
|
||||
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
"""Add answer-level feedback reasons and agent score dimensions.
|
||||
|
||||
Revision ID: 20260916_02
|
||||
Revises: 20260916_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_02"
|
||||
down_revision = "20260916_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "feedback"
|
||||
_OLD_INDEX = "uq_feedback_thread_run_user"
|
||||
_TARGET_INDEX = "uq_feedback_target_key"
|
||||
_AGENT_INDEX = "ix_feedback_agent_created"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
return {
|
||||
str(column["name"])
|
||||
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
|
||||
}
|
||||
|
||||
|
||||
def _indexes() -> set[str]:
|
||||
return {
|
||||
str(index["name"])
|
||||
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def _json_type() -> sa.types.TypeEngine:
|
||||
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
|
||||
|
||||
|
||||
def _empty_reasons_value() -> str | list[str]:
|
||||
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
|
||||
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
|
||||
# should receive the native list so their type processors serialize it.
|
||||
return "[]" if op.get_bind().dialect.name == "mysql" else []
|
||||
|
||||
|
||||
def _target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _backfill() -> None:
|
||||
bind = op.get_bind()
|
||||
feedback = sa.table(
|
||||
_TABLE,
|
||||
sa.column("feedback_id", sa.String(64)),
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("thread_id", sa.String(64)),
|
||||
sa.column("user_id", sa.String(64)),
|
||||
sa.column("message_id", sa.String(64)),
|
||||
sa.column("agent_id", sa.String(128)),
|
||||
sa.column("target_key", sa.String(64)),
|
||||
sa.column("reasons", _json_type()),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
rows = list(
|
||||
bind.execute(
|
||||
sa.select(
|
||||
feedback.c.feedback_id,
|
||||
feedback.c.run_id,
|
||||
feedback.c.thread_id,
|
||||
feedback.c.user_id,
|
||||
feedback.c.message_id,
|
||||
feedback.c.agent_id,
|
||||
feedback.c.created_at,
|
||||
)
|
||||
).mappings()
|
||||
)
|
||||
|
||||
run_agents: dict[str, str] = {}
|
||||
inspector = sa.inspect(bind)
|
||||
if rows and inspector.has_table("runs"):
|
||||
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
|
||||
if "assistant_id" in run_columns:
|
||||
runs = sa.table(
|
||||
"runs",
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("assistant_id", sa.String(128)),
|
||||
)
|
||||
run_ids = sorted({str(row["run_id"]) for row in rows})
|
||||
run_agents = {
|
||||
str(row.run_id): str(row.assistant_id)
|
||||
for row in bind.execute(
|
||||
sa.select(runs.c.run_id, runs.c.assistant_id).where(
|
||||
runs.c.run_id.in_(run_ids)
|
||||
)
|
||||
)
|
||||
if row.assistant_id
|
||||
}
|
||||
|
||||
seen_target_keys: set[str] = set()
|
||||
for row in rows:
|
||||
agent_id = str(
|
||||
run_agents.get(str(row["run_id"]))
|
||||
or row.get("agent_id")
|
||||
or "lead_agent"
|
||||
)[:128]
|
||||
target_key = _target_key(
|
||||
user_id=row.get("user_id"),
|
||||
thread_id=str(row["thread_id"]),
|
||||
run_id=str(row["run_id"]),
|
||||
message_id=row.get("message_id"),
|
||||
agent_id=agent_id,
|
||||
)
|
||||
# SQL unique indexes historically allowed several NULL-user rows.
|
||||
# Keep those legacy records without blocking the new unique digest;
|
||||
# one canonical row retains the deterministic future-upsert key.
|
||||
if target_key in seen_target_keys:
|
||||
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
|
||||
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
seen_target_keys.add(target_key)
|
||||
bind.execute(
|
||||
sa.update(feedback)
|
||||
.where(feedback.c.feedback_id == row["feedback_id"])
|
||||
.values(
|
||||
agent_id=agent_id,
|
||||
target_key=target_key,
|
||||
reasons=_empty_reasons_value(),
|
||||
updated_at=row.get("created_at"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return
|
||||
|
||||
columns = _columns()
|
||||
if "agent_id" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
"agent_id",
|
||||
sa.String(length=128),
|
||||
nullable=True,
|
||||
server_default="lead_agent",
|
||||
),
|
||||
)
|
||||
if "target_key" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("target_key", sa.String(length=64), nullable=True),
|
||||
)
|
||||
if "reasons" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("reasons", _json_type(), nullable=True),
|
||||
)
|
||||
if "updated_at" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
_backfill()
|
||||
|
||||
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
|
||||
# native ALTER statements on MySQL/PostgreSQL where possible.
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"agent_id",
|
||||
existing_type=sa.String(length=128),
|
||||
nullable=False,
|
||||
server_default=None,
|
||||
)
|
||||
batch.alter_column(
|
||||
"target_key",
|
||||
existing_type=sa.String(length=64),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"reasons",
|
||||
existing_type=_json_type(),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"updated_at",
|
||||
existing_type=sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
indexes = _indexes()
|
||||
if _OLD_INDEX in indexes:
|
||||
op.drop_index(_OLD_INDEX, table_name=_TABLE)
|
||||
indexes = _indexes()
|
||||
if _TARGET_INDEX not in indexes:
|
||||
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
|
||||
if _AGENT_INDEX not in indexes:
|
||||
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
indexes = _indexes()
|
||||
if _AGENT_INDEX in indexes:
|
||||
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
|
||||
if _TARGET_INDEX in indexes:
|
||||
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
|
||||
if _OLD_INDEX not in _indexes():
|
||||
op.create_index(
|
||||
_OLD_INDEX,
|
||||
_TABLE,
|
||||
["thread_id", "run_id", "user_id"],
|
||||
unique=True,
|
||||
mysql_length={"user_id": 63},
|
||||
)
|
||||
columns = _columns()
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
for name in ("updated_at", "reasons", "target_key", "agent_id"):
|
||||
if name in columns:
|
||||
batch.drop_column(name)
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Persist the latest LLMWiki incremental-sync result.
|
||||
|
||||
Revision ID: 20260920_01
|
||||
Revises: 20260916_02
|
||||
Create Date: 2026-09-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260920_01"
|
||||
down_revision = "20260916_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
return
|
||||
columns = {
|
||||
str(column["name"])
|
||||
for column in inspector.get_columns("llmwiki_wiki_sync_states")
|
||||
}
|
||||
if "last_sync_result" not in columns:
|
||||
op.add_column(
|
||||
"llmwiki_wiki_sync_states",
|
||||
sa.Column("last_sync_result", PortableJSON(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_wiki_sync_states"):
|
||||
return
|
||||
columns = {
|
||||
str(column["name"])
|
||||
for column in inspector.get_columns("llmwiki_wiki_sync_states")
|
||||
}
|
||||
if "last_sync_result" in columns:
|
||||
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""Add editable starter questions and retrieval settings to LLMWiki mappings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260920_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_knowledge_bases"):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
columns = _columns()
|
||||
if not columns:
|
||||
return
|
||||
if "test_questions" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("test_questions", sa.JSON(), nullable=True))
|
||||
if "retrieval_settings" not in columns:
|
||||
op.add_column("llmwiki_knowledge_bases", sa.Column("retrieval_settings", sa.JSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
columns = _columns()
|
||||
if "retrieval_settings" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
|
||||
if "test_questions" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "test_questions")
|
||||
Reference in new issue
Block a user