feat: MySQL 索引长度兼容 + 启动自动补列 + 未发布徽标与查询加载态
后端: - 索引/唯一键改用 mysql_length 前缀索引,适配 MySQL 767 字节限制(error 1071):knowledge_notes、knowledge_entities、knowledge_graph_blacklist、agent_skills、feedback - engine.py 新增 _ensure_orm_columns_sync:启动时对比 ORM 元数据,自动 ALTER TABLE 补齐缺失的可空/带默认值列及其索引 - 新增 scripts/audit_index_key_lengths.py 审计全部索引键长度 - 更新 backend/CLAUDE.md 迁移说明 前端: - 智能体卡片/技能列表:所有者与管理员可见未发布徽标 - 智能体广场:服务端查询进行中时灰显旧结果并悬浮加载提示 - 排行榜入口从 Header 移到欢迎语旁 - threads/hooks.ts:抽取 updateCachedThreadLists 统一处理数组与分页两种缓存形态 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
da677f87d5
commit
57a87d6322
15 files changed
+330
-102
No files matched your search
@@ -330,7 +330,9 @@ Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` /
|
||||
|
||||
### Schema Migrations / Manual Column-Add SQL (`scripts/sql/`)
|
||||
|
||||
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup only `Base.metadata.create_all` runs, which creates **new tables** but never patches **existing** ones (no new columns, no new indexes). So a column added to an ORM model appears automatically on a brand-new deployment, but is **missing on any existing MySQL database** — and code reading/writing that column will error there until the column is added by hand.
|
||||
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup `Base.metadata.create_all` runs (creates **new tables** only), followed by **`_ensure_orm_columns_sync`** — a best-effort sweep that compares every existing table against the ORM metadata and `ALTER TABLE … ADD COLUMN`s any missing column that is nullable or carries a `server_default` (plus the indexes covering those new columns). It works on all backends (sqlite/mysql/postgres; column DDL compiled through the active dialect), so a simple add-column lands automatically on existing databases at the next restart. NOT NULL columns without a server_default, type changes, widenings, and constraint changes are **not** auto-patched — those still need the manual SQL below. Tests: `tests/test_ensure_orm_columns.py`.
|
||||
|
||||
**MySQL index-key limit (required):** every index/unique key must fit MySQL's legacy 767-byte InnoDB prefix limit assuming utf8mb4 (4 bytes/char) — i.e. ≤191 chars total per key. Columns longer than that must use an explicit prefix index (`Index(..., mysql_length=191)` or `mysql_length={"col": N}` for composites) instead of `index=True`/`UniqueConstraint`, otherwise `create_all` fails with error 1071 on older MySQL servers. `scripts/audit_index_key_lengths.py` audits the whole metadata; run it after adding any indexed string column.
|
||||
|
||||
**Convention (required):** whenever code adds a field to an existing table (a new `mapped_column` on an ORM model), you must, in the same change:
|
||||
1. Keep the matching alembic version file under `persistence/migrations/versions/` (guarded with `_has_table`/`_has_column`), and
|
||||
|
||||
@@ -13,25 +13,35 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
|
||||
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
|
||||
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
|
||||
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
|
||||
# with error 1071 ("Specified key was too long").
|
||||
|
||||
|
||||
class KnowledgeNoteRow(Base):
|
||||
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
|
||||
|
||||
__tablename__ = "knowledge_notes"
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
|
||||
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
content_md: Mapped[str] = mapped_column(PortableLongText())
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
|
||||
vault_path: Mapped[str | None] = mapped_column(String(1024))
|
||||
source_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
|
||||
source_id: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
|
||||
confidence: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
@@ -118,10 +128,15 @@ class KnowledgeEntityRow(Base):
|
||||
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
|
||||
|
||||
__tablename__ = "knowledge_entities"
|
||||
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
|
||||
__table_args__ = (
|
||||
Index("ix_knowledge_entities_name", "name", mysql_length=191),
|
||||
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
|
||||
# realistically far shorter than 120 chars, so the prefix is lossless.
|
||||
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256), index=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
entity_type: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
@@ -134,9 +149,10 @@ class KnowledgeGraphBlacklistRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_graph_blacklist"
|
||||
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
|
||||
label: Mapped[str] = mapped_column(String(512))
|
||||
created_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
+6
-2
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -86,7 +86,11 @@ class AgentSkillRow(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "agent_skills"
|
||||
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
|
||||
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
|
||||
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
|
||||
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
|
||||
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
@@ -194,6 +194,79 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
|
||||
``_run_alembic_upgrade`` call) — so on an existing database a new
|
||||
``mapped_column`` on an ORM model would otherwise error at first query
|
||||
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Works on sqlite/mysql/postgres — the
|
||||
column DDL is compiled through the active dialect, so portable types
|
||||
(``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the
|
||||
right backend type. Everything is per-column best-effort: a failure is
|
||||
logged and startup continues.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
from sqlalchemy.schema import CreateColumn
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
try:
|
||||
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
inspector = sa_inspect(sync_conn)
|
||||
existing_tables = set(inspector.get_table_names())
|
||||
preparer = sync_conn.dialect.identifier_preparer
|
||||
|
||||
for table in Base.metadata.sorted_tables:
|
||||
if table.name not in existing_tables:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
table.name,
|
||||
col.name,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
@@ -344,6 +417,15 @@ async def init_engine(
|
||||
else:
|
||||
raise
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
+7
-2
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String, Text, UniqueConstraint
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
@@ -14,7 +14,12 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Audit ORM index/unique-key byte sizes against MySQL InnoDB prefix limits.
|
||||
|
||||
MySQL with the legacy 767-byte InnoDB index-prefix limit (5.6, or 5.7 with
|
||||
COMPACT row format) rejects any index whose key exceeds 767 bytes — and
|
||||
utf8mb4 costs 4 bytes per char, so an indexed VARCHAR key may total at most
|
||||
191 chars. Modern servers (DYNAMIC row format) allow 3072 bytes = 768 chars.
|
||||
|
||||
Run: PYTHONPATH=. .venv/Scripts/python scripts/audit_index_key_lengths.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from sqlalchemy import String, UniqueConstraint
|
||||
|
||||
import deerflow.persistence.models # noqa: F401 (registers all ORM models)
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
LIMIT_OLD = 767
|
||||
LIMIT_NEW = 3072
|
||||
BYTES_PER_CHAR = 4 # utf8mb4
|
||||
|
||||
|
||||
def _col_key_bytes(col) -> int | None:
|
||||
t = col.type
|
||||
# unwrap TypeDecorator
|
||||
impl = getattr(t, "impl", None)
|
||||
if impl is not None:
|
||||
t = impl
|
||||
if isinstance(t, String):
|
||||
return (t.length or 255) * BYTES_PER_CHAR
|
||||
return 8 # ints/datetimes/floats — small fixed size, close enough
|
||||
|
||||
|
||||
def _key_bytes(col, mysql_length) -> int:
|
||||
full = _col_key_bytes(col) or 0
|
||||
if mysql_length is None:
|
||||
return full
|
||||
if isinstance(mysql_length, dict):
|
||||
prefix = mysql_length.get(col.name)
|
||||
else:
|
||||
prefix = mysql_length
|
||||
if prefix is None:
|
||||
return full
|
||||
return min(full, int(prefix) * BYTES_PER_CHAR)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bad = 0
|
||||
for table in Base.metadata.sorted_tables:
|
||||
keys: list[tuple[str, str, list, object]] = []
|
||||
for idx in table.indexes:
|
||||
keys.append(("INDEX", idx.name, list(idx.columns), idx.dialect_options["mysql"].get("length")))
|
||||
for cons in table.constraints:
|
||||
if isinstance(cons, UniqueConstraint):
|
||||
keys.append(("UNIQUE", cons.name or "?", list(cons.columns), None))
|
||||
for kind, name, cols, mysql_length in keys:
|
||||
total = sum(_key_bytes(c, mysql_length) for c in cols)
|
||||
if total > LIMIT_OLD:
|
||||
bad += 1
|
||||
tag = "EXCEEDS 3072 (fails on ALL MySQL)" if total > LIMIT_NEW else "exceeds 767 (fails on legacy MySQL)"
|
||||
detail = ", ".join(f"{c.name}({getattr(getattr(c.type, 'impl', c.type), 'length', '?')})" for c in cols)
|
||||
print(f"{table.name}: {kind} {name} = {total} bytes [{detail}] -- {tag}")
|
||||
if not bad:
|
||||
print("OK: every index key fits within 767 bytes (utf8mb4 worst case)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user