chore(persistence): 增加 Workflow Studio 表重建开关与迁移兼容修正

提交内容:为数据库生命周期增加按白名单清单重建 Workflow Studio 表的受限流程,限定仅 MySQL 且检查当前连接数据库名称一致,避免误删非目标 schema。

启动流程中新增 DEER_FLOW_REBUILD_WORKFLOW_SCHEMA 环境变量入口与持久层重建 API,并在 lifespan 里按需执行;导出到 persistence 包,便于后续调用与测试复用。

迁移与模型修复:将 workflow 相关大文本字段切到 MySQL LONGTEXT,不再给 TEXT/LONGTEXT 设置 server_default;新增 draft_canvas_schema_json 的迁移回填与非空约束变更,规避 MySQL 5.7 对默认值限制。

测试覆盖:补充 workflow 定义 ddl 长文本断言与 schema 重建保护性测试(仅允许清单表、数据库名不匹配直接拒绝),并把新测试文件加入版本追踪。
This commit is contained in:
董宏凯 committed 2026-08-31 16:45:06 +08:00
1 parent f0ddade8a7
commit ac92f29db1
10 files changed
+300 -39

No files matched your search

@@ -8,6 +8,18 @@ Usage:
from deerflow.persistence import init_engine, close_engine, get_session_factory
"""
from deerflow.persistence.engine import close_engine, get_engine, get_session_factory, init_engine
__all__ = ["close_engine", "get_engine", "get_session_factory", "init_engine"]
from deerflow.persistence.engine import (
close_engine,
get_engine,
get_session_factory,
init_engine,
rebuild_workflow_schema_for_mysql,
)
__all__ = [
"close_engine",
"get_engine",
"get_session_factory",
"init_engine",
"rebuild_workflow_schema_for_mysql",
]
@@ -38,7 +38,21 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
_engine_loop: asyncio.AbstractEventLoop | None = None
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
def _add_runtime_vendor_path() -> None:
@@ -527,9 +541,73 @@ def get_engine() -> AsyncEngine | None:
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def fire_and_forget_db_write(coro) -> None:
@@ -9,6 +9,7 @@ from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260828_01"
down_revision = "20260825_01"
@@ -19,16 +20,21 @@ depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
# Match PortableLongText used by the ORM. MySQL TEXT only holds 64 KiB,
# which is too small for serialized workflow graphs.
long_text = LONGTEXT() if bind.dialect.name == "mysql" else sa.Text()
if not inspector.has_table("workflow_definitions"):
op.create_table(
"workflow_definitions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("description", sa.Text(), nullable=False, server_default=""),
# Older MySQL versions reject defaults on TEXT/LONGTEXT columns.
# The repository always writes both fields explicitly.
sa.Column("description", sa.Text(), nullable=False),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
sa.Column("draft_graph_json", sa.Text(), nullable=False),
sa.Column("draft_graph_json", long_text, nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
@@ -41,13 +47,13 @@ def upgrade() -> None:
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_number", sa.Integer(), nullable=False),
sa.Column("graph_json", sa.Text(), nullable=False),
sa.Column("graph_json", long_text, nullable=False),
sa.Column("graph_hash", sa.String(length=64), nullable=False),
sa.Column("input_schema", sa.Text(), nullable=True),
sa.Column("output_schema", sa.Text(), nullable=True),
sa.Column("published_by", sa.String(length=64), nullable=True),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
sa.Column("change_note", sa.Text(), nullable=False),
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
)
@@ -14,6 +14,7 @@ from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.mysql import LONGTEXT
revision = "20260829_02"
down_revision = "20260829_01"
@@ -28,10 +29,30 @@ def upgrade() -> None:
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "draft_canvas_schema_json" not in existing:
# Follow roundtable/deep-research migrations: large JSON is LONGTEXT on
# MySQL, and we must not use a TEXT/LONGTEXT server default (unsupported
# by MySQL 5.7). Add it nullable, backfill legacy rows, then enforce the
# ORM's NOT NULL contract.
canvas_type = LONGTEXT() if bind.dialect.name == "mysql" else sa.Text()
op.add_column(
"workflow_definitions",
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
sa.Column("draft_canvas_schema_json", canvas_type, nullable=True),
)
op.execute(
sa.text(
"UPDATE workflow_definitions "
"SET draft_canvas_schema_json = '{}' "
"WHERE draft_canvas_schema_json IS NULL"
)
)
# ``batch_alter_table`` uses a native ALTER TABLE on MySQL while also
# handling SQLite's lack of ALTER COLUMN for legacy SQLite deployments.
with op.batch_alter_table("workflow_definitions") as batch_op:
batch_op.alter_column(
"draft_canvas_schema_json",
existing_type=canvas_type,
nullable=False,
)
if "published_canvas_schema_hash" not in existing:
op.add_column(
"workflow_definitions",
@@ -33,7 +33,10 @@ class WorkflowDefinitionRow(Base):
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
# the executable WorkflowGraph lives in draft_graph_json above.
draft_canvas_schema_json: Mapped[str] = mapped_column(
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
# Keep this as an ORM-side default only. MySQL 5.7 rejects DEFAULT on
# TEXT/LONGTEXT columns, while workflow creation always supplies this
# value explicitly as well.
PortableLongText(), default="{}", nullable=False
)
# sha256 of the canvas document the current published version was compiled
# from (audit trail only; publish always freezes the execution graph).