feat: MySQL 索引长度兼容 + 启动自动补列 + 未发布徽标与查询加载态

后端:
- 索引/唯一键改用 mysql_length 前缀索引,适配 MySQL 767 字节限制(error 1071):knowledge_notes、knowledge_entities、knowledge_graph_blacklist、agent_skills、feedback
- engine.py 新增 _ensure_orm_columns_sync:启动时对比 ORM 元数据,自动 ALTER TABLE 补齐缺失的可空/带默认值列及其索引
- 新增 scripts/audit_index_key_lengths.py 审计全部索引键长度
- 更新 backend/CLAUDE.md 迁移说明

前端:
- 智能体卡片/技能列表:所有者与管理员可见未发布徽标
- 智能体广场:服务端查询进行中时灰显旧结果并悬浮加载提示
- 排行榜入口从 Header 移到欢迎语旁
- threads/hooks.ts:抽取 updateCachedThreadLists 统一处理数组与分页两种缓存形态

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Fable 5 committed 2026-06-12 14:51:50 +08:00
1 parent da677f87d5
commit 57a87d6322
15 files changed
+330 -102

No files matched your search

+3 -1
View File
@@ -330,7 +330,9 @@ Two `TypeDecorator`s keep ORM columns correct across the `sqlite` / `postgres` /
### Schema Migrations / Manual Column-Add SQL (`scripts/sql/`)
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup only `Base.metadata.create_all` runs, which creates **new tables** but never patches **existing** ones (no new columns, no new indexes). So a column added to an ORM model appears automatically on a brand-new deployment, but is **missing on any existing MySQL database** — and code reading/writing that column will error there until the column is added by hand.
Alembic auto-migration is **disabled** — `_run_alembic_upgrade(url)` is commented out in `persistence/engine.py`. On startup `Base.metadata.create_all` runs (creates **new tables** only), followed by **`_ensure_orm_columns_sync`** — a best-effort sweep that compares every existing table against the ORM metadata and `ALTER TABLE … ADD COLUMN`s any missing column that is nullable or carries a `server_default` (plus the indexes covering those new columns). It works on all backends (sqlite/mysql/postgres; column DDL compiled through the active dialect), so a simple add-column lands automatically on existing databases at the next restart. NOT NULL columns without a server_default, type changes, widenings, and constraint changes are **not** auto-patched — those still need the manual SQL below. Tests: `tests/test_ensure_orm_columns.py`.
**MySQL index-key limit (required):** every index/unique key must fit MySQL's legacy 767-byte InnoDB prefix limit assuming utf8mb4 (4 bytes/char) — i.e. ≤191 chars total per key. Columns longer than that must use an explicit prefix index (`Index(..., mysql_length=191)` or `mysql_length={"col": N}` for composites) instead of `index=True`/`UniqueConstraint`, otherwise `create_all` fails with error 1071 on older MySQL servers. `scripts/audit_index_key_lengths.py` audits the whole metadata; run it after adding any indexed string column.
**Convention (required):** whenever code adds a field to an existing table (a new `mapped_column` on an ORM model), you must, in the same change:
1. Keep the matching alembic version file under `persistence/migrations/versions/` (guarded with `_has_table`/`_has_column`), and
@@ -13,25 +13,35 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
# MySQL InnoDB with the legacy 767-byte index-prefix limit (5.6, or 5.7 with
# COMPACT row format) allows at most 191 utf8mb4 chars per index key. Columns
# longer than that use an explicit prefix index (``mysql_length`` — ignored by
# SQLite/Postgres) instead of ``index=True``, otherwise ``create_all`` fails
# with error 1071 ("Specified key was too long").
class KnowledgeNoteRow(Base):
"""Knowledge note main table (mirrors ``knowledge_notes``)."""
__tablename__ = "knowledge_notes"
__table_args__ = (
Index("ix_knowledge_notes_vault_path", "vault_path", mysql_length=191),
Index("ix_knowledge_notes_source_id", "source_id", mysql_length=191),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
title: Mapped[str] = mapped_column(String(512))
summary: Mapped[str | None] = mapped_column(Text)
content_md: Mapped[str] = mapped_column(PortableLongText())
vault_path: Mapped[str | None] = mapped_column(String(1024), index=True)
vault_path: Mapped[str | None] = mapped_column(String(1024))
source_type: Mapped[str] = mapped_column(String(64), index=True)
source_id: Mapped[str | None] = mapped_column(String(256), index=True)
source_id: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(32), default="approved", index=True)
confidence: Mapped[float] = mapped_column(Float, default=0.0)
created_by: Mapped[str | None] = mapped_column(String(128), index=True)
@@ -118,10 +128,15 @@ class KnowledgeEntityRow(Base):
"""Knowledge entity (phase 4, mirrors ``knowledge_entities``)."""
__tablename__ = "knowledge_entities"
__table_args__ = (UniqueConstraint("name", "entity_type", name="uq_knowledge_entities_name_type"),)
__table_args__ = (
Index("ix_knowledge_entities_name", "name", mysql_length=191),
# Prefix-unique: (120 + 64) * 4 = 736 bytes <= 767. Entity names are
# realistically far shorter than 120 chars, so the prefix is lossless.
Index("uq_knowledge_entities_name_type", "name", "entity_type", unique=True, mysql_length={"name": 120}),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(256), index=True)
name: Mapped[str] = mapped_column(String(256))
entity_type: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -134,9 +149,10 @@ class KnowledgeGraphBlacklistRow(Base):
"""
__tablename__ = "knowledge_graph_blacklist"
__table_args__ = (Index("ix_knowledge_graph_blacklist_label", "label", unique=True, mysql_length=191),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
label: Mapped[str] = mapped_column(String(512), unique=True, index=True)
label: Mapped[str] = mapped_column(String(512))
created_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Integer, String, UniqueConstraint
from sqlalchemy import BigInteger, Boolean, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -86,7 +86,11 @@ class AgentSkillRow(Base):
"""
__tablename__ = "agent_skills"
__table_args__ = (UniqueConstraint("agent_id", "skill_name", name="uq_agent_skills_agent_skill"),)
# Prefix-unique instead of UniqueConstraint: (64 + 120) * 4 = 736 bytes
# fits MySQL's legacy 767-byte index limit (utf8mb4); a plain unique over
# agent_id + skill_name(191) would be 1020 bytes and fail create_all with
# error 1071 there. mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_agent_skills_agent_skill", "agent_id", "skill_name", unique=True, mysql_length={"skill_name": 120}),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
agent_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -194,6 +194,79 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
existing ones, and Alembic auto-upgrade is disabled (see the commented-out
``_run_alembic_upgrade`` call) — so on an existing database a new
``mapped_column`` on an ORM model would otherwise error at first query
(e.g. ``no such column: roundtable_drafts.task_id`` on SQLite).
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Indexes covering a newly
added column are created as well. Works on sqlite/mysql/postgres — the
column DDL is compiled through the active dialect, so portable types
(``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the
right backend type. Everything is per-column best-effort: a failure is
logged and startup continues.
"""
from sqlalchemy import inspect as sa_inspect
from sqlalchemy.schema import CreateColumn
from deerflow.persistence.base import Base
try:
import deerflow.persistence.models # noqa: F401 — register all ORM models on Base.metadata
except ImportError:
pass
inspector = sa_inspect(sync_conn)
existing_tables = set(inspector.get_table_names())
preparer = sync_conn.dialect.identifier_preparer
for table in Base.metadata.sorted_tables:
if table.name not in existing_tables:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
added: set[str] = set()
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
table.name,
col.name,
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
added.add(col.name)
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
if not added:
continue
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
@@ -344,6 +417,15 @@ async def init_engine(
else:
raise
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
except Exception:
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String, Text, UniqueConstraint
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -14,7 +14,12 @@ from deerflow.persistence.types import BeijingDateTime
class FeedbackRow(Base):
__tablename__ = "feedback"
__table_args__ = (UniqueConstraint("thread_id", "run_id", "user_id", name="uq_feedback_thread_run_user"),)
# Prefix-unique instead of UniqueConstraint: a plain unique over three
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
# mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
@@ -0,0 +1,71 @@
"""Audit ORM index/unique-key byte sizes against MySQL InnoDB prefix limits.
MySQL with the legacy 767-byte InnoDB index-prefix limit (5.6, or 5.7 with
COMPACT row format) rejects any index whose key exceeds 767 bytes — and
utf8mb4 costs 4 bytes per char, so an indexed VARCHAR key may total at most
191 chars. Modern servers (DYNAMIC row format) allow 3072 bytes = 768 chars.
Run: PYTHONPATH=. .venv/Scripts/python scripts/audit_index_key_lengths.py
"""
from __future__ import annotations
import sys
from sqlalchemy import String, UniqueConstraint
import deerflow.persistence.models # noqa: F401 (registers all ORM models)
from deerflow.persistence.base import Base
LIMIT_OLD = 767
LIMIT_NEW = 3072
BYTES_PER_CHAR = 4 # utf8mb4
def _col_key_bytes(col) -> int | None:
t = col.type
# unwrap TypeDecorator
impl = getattr(t, "impl", None)
if impl is not None:
t = impl
if isinstance(t, String):
return (t.length or 255) * BYTES_PER_CHAR
return 8 # ints/datetimes/floats — small fixed size, close enough
def _key_bytes(col, mysql_length) -> int:
full = _col_key_bytes(col) or 0
if mysql_length is None:
return full
if isinstance(mysql_length, dict):
prefix = mysql_length.get(col.name)
else:
prefix = mysql_length
if prefix is None:
return full
return min(full, int(prefix) * BYTES_PER_CHAR)
def main() -> int:
bad = 0
for table in Base.metadata.sorted_tables:
keys: list[tuple[str, str, list, object]] = []
for idx in table.indexes:
keys.append(("INDEX", idx.name, list(idx.columns), idx.dialect_options["mysql"].get("length")))
for cons in table.constraints:
if isinstance(cons, UniqueConstraint):
keys.append(("UNIQUE", cons.name or "?", list(cons.columns), None))
for kind, name, cols, mysql_length in keys:
total = sum(_key_bytes(c, mysql_length) for c in cols)
if total > LIMIT_OLD:
bad += 1
tag = "EXCEEDS 3072 (fails on ALL MySQL)" if total > LIMIT_NEW else "exceeds 767 (fails on legacy MySQL)"
detail = ", ".join(f"{c.name}({getattr(getattr(c.type, 'impl', c.type), 'length', '?')})" for c in cols)
print(f"{table.name}: {kind} {name} = {total} bytes [{detail}] -- {tag}")
if not bad:
print("OK: every index key fits within 767 bytes (utf8mb4 worst case)")
return 0
if __name__ == "__main__":
sys.exit(main())