sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -0,0 +1,21 @@
"""Merge assistant-wiki metadata and report-collaboration heads.
Revision ID: 20260909_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-09
"""
from __future__ import annotations
revision = "20260909_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,74 @@
"""Create durable public-document library tables.
Revision ID: 20260914_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260914_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("gongwen_documents"):
op.create_table(
"gongwen_documents",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("user_id", sa.String(length=64), nullable=True),
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
sa.Column("template_key", sa.String(length=96), nullable=True),
sa.Column("research_session_id", sa.String(length=64), nullable=True),
sa.Column("original_version_id", sa.String(length=64), nullable=True),
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
sa.Column("issue_counts", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
if not inspector.has_table("gongwen_document_versions"):
op.create_table(
"gongwen_document_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("document_id", sa.String(length=64), nullable=False),
sa.Column("version_no", sa.Integer(), nullable=False),
sa.Column("kind", sa.String(length=32), nullable=False),
sa.Column("file_name", sa.String(length=512), nullable=False),
sa.Column("storage_key", sa.String(length=1024), nullable=False),
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
sa.Column("review_snapshot", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
)
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
op.create_index(
"ix_gongwen_document_versions_document_created",
"gongwen_document_versions",
["document_id", "created_at"],
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("gongwen_document_versions"):
op.drop_table("gongwen_document_versions")
if inspector.has_table("gongwen_documents"):
op.drop_table("gongwen_documents")
@@ -0,0 +1,88 @@
"""Persist mapping-scoped links for local LLMWiki graph expansion.
Revision ID: 20260916_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260916_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("llmwiki_wiki_edges"):
return
op.create_table(
"llmwiki_wiki_edges",
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column(
"knowledge_base_mapping_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"source_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"target_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"edge_type",
sa.String(length=32),
nullable=False,
server_default="wiki_link",
),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
)
op.create_index(
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_source_page_id",
"llmwiki_wiki_edges",
["source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_target_page_id",
"llmwiki_wiki_edges",
["target_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_source",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_target",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "target_page_id"],
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
op.drop_table("llmwiki_wiki_edges")
@@ -0,0 +1,27 @@
"""Merge the public-document and assistant-knowledge migration heads.
Revision ID: 20260916_01
Revises: 20260909_01, 20260914_01
Create Date: 2026-09-16
The feedback-scoring migration was added as ``20260916_02`` and references
this merge revision. Keeping the merge as an explicit no-op makes the
Alembic graph linear again without changing either existing schema branch.
"""
from __future__ import annotations
revision = "20260916_01"
down_revision = ("20260909_01", "20260914_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Record the merge; both parent revisions already own their DDL."""
def downgrade() -> None:
"""Splitting the migration history requires no schema change."""
@@ -0,0 +1,240 @@
"""Add answer-level feedback reasons and agent score dimensions.
Revision ID: 20260916_02
Revises: 20260916_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import hashlib
import sqlalchemy as sa
from alembic import op
revision = "20260916_02"
down_revision = "20260916_01"
branch_labels = None
depends_on = None
_TABLE = "feedback"
_OLD_INDEX = "uq_feedback_thread_run_user"
_TARGET_INDEX = "uq_feedback_target_key"
_AGENT_INDEX = "ix_feedback_agent_created"
def _columns() -> set[str]:
return {
str(column["name"])
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
}
def _indexes() -> set[str]:
return {
str(index["name"])
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
if index.get("name")
}
def _json_type() -> sa.types.TypeEngine:
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
def _empty_reasons_value() -> str | list[str]:
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
# should receive the native list so their type processors serialize it.
return "[]" if op.get_bind().dialect.name == "mysql" else []
def _target_key(
*,
user_id: str | None,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
) -> str:
message_key = message_id or f"run:{run_id}"
raw = "\x1f".join(
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _backfill() -> None:
bind = op.get_bind()
feedback = sa.table(
_TABLE,
sa.column("feedback_id", sa.String(64)),
sa.column("run_id", sa.String(64)),
sa.column("thread_id", sa.String(64)),
sa.column("user_id", sa.String(64)),
sa.column("message_id", sa.String(64)),
sa.column("agent_id", sa.String(128)),
sa.column("target_key", sa.String(64)),
sa.column("reasons", _json_type()),
sa.column("created_at", sa.DateTime(timezone=True)),
sa.column("updated_at", sa.DateTime(timezone=True)),
)
rows = list(
bind.execute(
sa.select(
feedback.c.feedback_id,
feedback.c.run_id,
feedback.c.thread_id,
feedback.c.user_id,
feedback.c.message_id,
feedback.c.agent_id,
feedback.c.created_at,
)
).mappings()
)
run_agents: dict[str, str] = {}
inspector = sa.inspect(bind)
if rows and inspector.has_table("runs"):
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
if "assistant_id" in run_columns:
runs = sa.table(
"runs",
sa.column("run_id", sa.String(64)),
sa.column("assistant_id", sa.String(128)),
)
run_ids = sorted({str(row["run_id"]) for row in rows})
run_agents = {
str(row.run_id): str(row.assistant_id)
for row in bind.execute(
sa.select(runs.c.run_id, runs.c.assistant_id).where(
runs.c.run_id.in_(run_ids)
)
)
if row.assistant_id
}
seen_target_keys: set[str] = set()
for row in rows:
agent_id = str(
run_agents.get(str(row["run_id"]))
or row.get("agent_id")
or "lead_agent"
)[:128]
target_key = _target_key(
user_id=row.get("user_id"),
thread_id=str(row["thread_id"]),
run_id=str(row["run_id"]),
message_id=row.get("message_id"),
agent_id=agent_id,
)
# SQL unique indexes historically allowed several NULL-user rows.
# Keep those legacy records without blocking the new unique digest;
# one canonical row retains the deterministic future-upsert key.
if target_key in seen_target_keys:
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
else:
seen_target_keys.add(target_key)
bind.execute(
sa.update(feedback)
.where(feedback.c.feedback_id == row["feedback_id"])
.values(
agent_id=agent_id,
target_key=target_key,
reasons=_empty_reasons_value(),
updated_at=row.get("created_at"),
)
)
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return
columns = _columns()
if "agent_id" not in columns:
op.add_column(
_TABLE,
sa.Column(
"agent_id",
sa.String(length=128),
nullable=True,
server_default="lead_agent",
),
)
if "target_key" not in columns:
op.add_column(
_TABLE,
sa.Column("target_key", sa.String(length=64), nullable=True),
)
if "reasons" not in columns:
op.add_column(
_TABLE,
sa.Column("reasons", _json_type(), nullable=True),
)
if "updated_at" not in columns:
op.add_column(
_TABLE,
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
)
_backfill()
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
# native ALTER statements on MySQL/PostgreSQL where possible.
with op.batch_alter_table(_TABLE) as batch:
batch.alter_column(
"agent_id",
existing_type=sa.String(length=128),
nullable=False,
server_default=None,
)
batch.alter_column(
"target_key",
existing_type=sa.String(length=64),
nullable=False,
)
batch.alter_column(
"reasons",
existing_type=_json_type(),
nullable=False,
)
batch.alter_column(
"updated_at",
existing_type=sa.DateTime(timezone=True),
nullable=False,
)
indexes = _indexes()
if _OLD_INDEX in indexes:
op.drop_index(_OLD_INDEX, table_name=_TABLE)
indexes = _indexes()
if _TARGET_INDEX not in indexes:
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
if _AGENT_INDEX not in indexes:
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
def downgrade() -> None:
if not sa.inspect(op.get_bind()).has_table(_TABLE):
return
indexes = _indexes()
if _AGENT_INDEX in indexes:
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
if _TARGET_INDEX in indexes:
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
if _OLD_INDEX not in _indexes():
op.create_index(
_OLD_INDEX,
_TABLE,
["thread_id", "run_id", "user_id"],
unique=True,
mysql_length={"user_id": 63},
)
columns = _columns()
with op.batch_alter_table(_TABLE) as batch:
for name in ("updated_at", "reasons", "target_key", "agent_id"):
if name in columns:
batch.drop_column(name)
@@ -0,0 +1,45 @@
"""Persist the latest LLMWiki incremental-sync result.
Revision ID: 20260920_01
Revises: 20260916_02
Create Date: 2026-09-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260920_01"
down_revision = "20260916_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_wiki_sync_states"):
return
columns = {
str(column["name"])
for column in inspector.get_columns("llmwiki_wiki_sync_states")
}
if "last_sync_result" not in columns:
op.add_column(
"llmwiki_wiki_sync_states",
sa.Column("last_sync_result", PortableJSON(), nullable=True),
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_wiki_sync_states"):
return
columns = {
str(column["name"])
for column in inspector.get_columns("llmwiki_wiki_sync_states")
}
if "last_sync_result" in columns:
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
@@ -0,0 +1,36 @@
"""Add editable starter questions and retrieval settings to LLMWiki mappings."""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260920_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_knowledge_bases"):
return set()
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
def upgrade() -> None:
columns = _columns()
if not columns:
return
if "test_questions" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("test_questions", sa.JSON(), nullable=True))
if "retrieval_settings" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("retrieval_settings", sa.JSON(), nullable=True))
def downgrade() -> None:
columns = _columns()
if "retrieval_settings" in columns:
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
if "test_questions" in columns:
op.drop_column("llmwiki_knowledge_bases", "test_questions")