sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -11,14 +11,20 @@ None and fall back to in-memory implementations.
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import re
import sys
import time
import weakref
from contextvars import ContextVar, Token
from importlib.util import find_spec
from pathlib import Path
from typing import Any
from sqlalchemy import text
from sqlalchemy import event, text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
@@ -38,21 +44,167 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
# "got Future attached to a different loop". We capture the engine's home loop here so
# best-effort writes coming from worker threads / other loops can be routed back to it
# (see ``fire_and_forget_db_write``).
_engine_loop: asyncio.AbstractEventLoop | None = None
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
_engine_loop: asyncio.AbstractEventLoop | None = None
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
def bind_performance_request(request_id: str) -> Token:
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
def reset_performance_request(token: Token) -> None:
_performance_request_context.reset(token)
def _current_performance_request_id() -> str:
context = _performance_request_context.get()
if context is None:
return "-"
try:
task = asyncio.current_task()
except RuntimeError:
task = None
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
def _slow_sql_threshold_ms() -> float:
try:
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
except (TypeError, ValueError):
return 500.0
def _sql_operation(statement: str) -> str:
parts = statement.lstrip().split(None, 1)
candidate = parts[0].upper() if parts else "EMPTY"
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
def _sql_table(statement: str) -> str:
"""Return the target/source table only, without query text or values."""
match = re.search(
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
statement,
re.IGNORECASE,
)
if match is None:
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
if match is None:
return "unknown"
return match.group(1).strip("`\".")[-128:] or "unknown"
def _sql_fingerprint(statement: str) -> str:
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
"""Extract safe driver diagnostics without logging SQL or bound values."""
if exception is None:
return "unknown", "-", "-"
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
args = getattr(exception, "args", ())
if code is None and args and isinstance(args[0], (int, str)):
candidate = str(args[0])
if candidate.isdigit():
code = candidate
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
diagnostic = getattr(exception, "diag", None)
constraint = getattr(diagnostic, "constraint_name", None)
def _safe_identifier(value: Any) -> str:
text_value = str(value or "")
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
"""Log slow SQL timings without logging query text or bound values."""
sync_engine = engine.sync_engine
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
if context is not None:
context._deerflow_perf_started = time.perf_counter()
context._deerflow_perf_operation = _sql_operation(statement)
context._deerflow_perf_statement = statement
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
if context is None:
return
started = getattr(context, "_deerflow_perf_started", None)
if started is None:
return
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
if elapsed_ms >= _slow_sql_threshold_ms():
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
logger.warning(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER"),
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
elapsed_ms,
bool(executemany),
fingerprint,
)
def handle_error(exception_context):
context = exception_context.execution_context
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
original_exception = exception_context.original_exception
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
table == "scheduled_task_runs"
and "UNIQUE constraint failed" in str(original_exception or "")
)
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
log(
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
_current_performance_request_id(),
backend,
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
table,
elapsed_ms,
bool(getattr(context, "executemany", False)) if context is not None else False,
fingerprint,
outcome,
type(original_exception).__name__ if original_exception is not None else "unknown",
error_code,
sqlstate,
constraint,
)
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
event.listen(sync_engine, "handle_error", handle_error)
# These are the complete workflow-studio persistence tables owned by DeerFlow.
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
# wildcard: another application might share a schema and have its own table
# with that prefix.
_WORKFLOW_SCHEMA_TABLE_NAMES = (
"workflow_definitions",
"workflow_versions",
"workflow_runs",
"workflow_node_runs",
"workflow_run_events",
"workflow_run_artifacts",
"workflow_data_sources",
)
def _add_runtime_vendor_path() -> None:
@@ -81,6 +233,42 @@ def _mysql_driver_name(url: str) -> str:
return "asyncmy"
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
kwargs = dict(
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
# These are not MySQL/psycopg connection arguments.
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
kwargs["connect_args"] = {"command_timeout": command_timeout}
return kwargs
async def _run_alembic_upgrade(url: str) -> None:
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
@@ -176,7 +364,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -215,7 +403,7 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
@@ -226,14 +414,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -258,10 +446,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
continue
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
missing = [c for c in table.columns if c.name not in existing_cols]
if not missing:
continue
for col in missing:
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
@@ -270,21 +456,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if not idx.name or idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
@@ -328,71 +514,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
def _acquire_gauss_mysql_schema_bootstrap_lock():
"""Serialize Dolphin schema bootstrap across workers on this host."""
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if not is_gauss_mysql_product():
return None
try:
import fcntl
except ImportError:
# Production offline deployments are Linux. Keeping a no-lock fallback
# makes imports and unit tests work on Windows without changing MySQL.
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
return None
try:
from deerflow.config.runtime_paths import runtime_home
home = runtime_home()
except Exception:
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+", encoding="utf-8")
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
return handle
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
if handle is None:
return
import fcntl
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
finally:
handle.close()
async def init_engine(
backend: str,
*,
@@ -401,28 +627,40 @@ async def init_engine(
pool_size: int = 5,
max_overflow: int = 10,
pool_timeout: int = 30,
pool_recycle: int = 1800,
pool_recycle: int | None = None,
command_timeout: float | None = 30,
sqlite_dir: str = "",
) -> None:
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
global _engine, _session_factory, _engine_loop
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if pool_recycle is None:
pool_recycle = 300 if backend == "postgres" else 1800
if backend == "memory":
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
@@ -431,25 +669,25 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend in {"postgres", "gauss"}:
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -459,12 +697,12 @@ async def init_engine(
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if gauss_mysql:
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
install_gauss_mysql_dialect_compatibility(driver)
if backend == "sqlite":
import os
from sqlalchemy import event
os.makedirs(sqlite_dir or ".", exist_ok=True)
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
@@ -489,31 +727,28 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend in {"postgres", "gauss"}:
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
elif backend == "mysql":
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
else:
raise ValueError(f"Unknown persistence backend: {backend!r}")
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
# Auto-create tables for fresh deployments, then let Alembic apply any
@@ -530,16 +765,32 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
await conn.run_sync(Base.metadata.create_all)
if gauss_mysql:
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
# Schema bootstrap is deliberately best-effort. A single incompatible or
# permission-restricted table must not make the whole Gateway unavailable:
# routes backed by healthy existing tables can still work, while a route
# whose table is missing will report its own request-time error. Keep the
# auto-create-database retries, but contain failures from both the initial
# create_all and the retry path.
schema_bootstrap_lock = None
if gauss_mysql:
try:
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
except Exception:
# A lock-file permission problem is operationally important, but
# the global startup invariant still requires the Gateway to come
# up and report schema errors at the affected routes.
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
try:
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await _create_all_with_product_integrity(conn)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
@@ -548,35 +799,33 @@ async def init_engine(
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
await _create_all_with_product_integrity(conn)
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
**_pooled_engine_kwargs(
backend=backend, url=url, echo=echo, pool_size=pool_size,
max_overflow=max_overflow, pool_timeout=pool_timeout,
pool_recycle=pool_recycle, command_timeout=command_timeout,
),
)
_install_slow_sql_logging(_engine, backend)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await _create_all_with_product_integrity(conn)
else:
raise
except Exception:
@@ -584,22 +833,34 @@ async def init_engine(
"Automatic ORM schema creation failed; continuing Gateway startup "
"with the existing database schema"
)
finally:
try:
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
except Exception:
logger.exception("Could not release openGauss schema bootstrap lock")
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
if gauss_mysql:
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
# aborting the transaction with error 322. The native-catalog helper
# above is the only schema repair path for this product.
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
else:
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -614,14 +875,17 @@ async def init_engine_from_config(config) -> None:
pool_size=config.pool_size,
max_overflow=getattr(config, "max_overflow", 10),
pool_timeout=getattr(config, "pool_timeout", 30),
pool_recycle=getattr(config, "pool_recycle", 1800),
pool_recycle=getattr(config, "pool_recycle", None),
command_timeout=getattr(config, "command_timeout", 30),
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
)
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
"""Return the async session factory, or None if backend=memory."""
return _session_factory
from deerflow.persistence.loop_isolation import isolated_session_factory
return isolated_session_factory() or _session_factory
def get_engine() -> AsyncEngine | None:
@@ -629,73 +893,73 @@ def get_engine() -> AsyncEngine | None:
return _engine
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
"""Return the event loop that owns the engine's connection pool (or None)."""
return _engine_loop
async def rebuild_workflow_schema_for_mysql() -> list[str]:
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
This is a deliberately narrow recovery operation for an unused Workflow
Studio installation. It never uses a prefix wildcard, never changes the
selected database, and refuses to run unless the engine is connected to
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
Returns the allow-listed workflow tables that existed before the rebuild.
"""
engine = get_engine()
if engine is None:
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
if engine.dialect.name != "mysql":
raise RuntimeError(
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
)
expected_database = engine.url.database
if not expected_database:
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
# Make the operation safe when this helper is invoked independently from
# normal app startup as well.
import deerflow.persistence.models # noqa: F401 — register ORM models
from deerflow.persistence.base import Base
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
if missing_models:
raise RuntimeError(
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
)
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
def _rebuild(sync_conn) -> list[str]:
from sqlalchemy import inspect as sa_inspect
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
if current_database != expected_database:
raise RuntimeError(
"Refusing workflow schema rebuild: connected database "
f"{current_database!r} does not match configured database {expected_database!r}"
)
existing = set(sa_inspect(sync_conn).get_table_names())
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
# SQLAlchemy orders parent/child tables around the foreign key from
# workflow_versions to workflow_definitions. Only tables in the exact
# allow-list above are ever passed to drop_all/create_all.
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
return present
async with engine.begin() as conn:
rebuilt = await conn.run_sync(_rebuild)
logger.warning(
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
expected_database,
", ".join(rebuilt) if rebuilt else "none (created fresh)",
)
return rebuilt
def fire_and_forget_db_write(coro) -> None:
@@ -744,10 +1008,13 @@ def _run_coro_on_throwaway_loop(coro):
Only reached when there is no usable engine home loop, or when the caller is
*synchronously* inside the home loop (routing back would deadlock). This
touches the shared connection pool from a throwaway loop, so it can still leak
a dead-loop-bound connection — but the common worker-thread path no longer
gets here (see ``run_db_blocking``).
uses an isolated, non-pooled engine for compatibility reads. The coroutine
must obtain its session factory inside its async body so the isolated
context applies; no connection is returned to the Gateway pool.
"""
from deerflow.persistence.loop_isolation import run_isolated_db_read
coro = run_isolated_db_read(coro)
try:
asyncio.get_running_loop()
in_loop = True