sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -215,7 +215,7 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
@@ -226,8 +226,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Indexes covering a newly
added column are created as well. Table-level ``UniqueConstraint`` objects
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
@@ -261,8 +261,7 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
if not missing:
continue
added: set[str] = set()
for col in missing:
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
@@ -271,24 +270,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
added.add(col.name)
logger.info("Auto-added missing column %s.%s", table.name, col.name)
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
if not added:
continue
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
continue
try:
idx.create(sync_conn)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
@@ -332,15 +328,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
def _acquire_gauss_mysql_schema_bootstrap_lock():
"""Serialize Dolphin schema bootstrap across workers on this host."""
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if not is_gauss_mysql_product():
return None
try:
import fcntl
except ImportError:
# Production offline deployments are Linux. Keeping a no-lock fallback
# makes imports and unit tests work on Windows without changing MySQL.
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
return None
try:
from deerflow.config.runtime_paths import runtime_home
home = runtime_home()
except Exception:
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+", encoding="utf-8")
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
return handle
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
if handle is None:
return
import fcntl
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
finally:
handle.close()
async def init_engine(
backend: str,
*,
@@ -371,6 +463,10 @@ async def init_engine(
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
@@ -394,6 +490,10 @@ async def init_engine(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -403,6 +503,10 @@ async def init_engine(
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if gauss_mysql:
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
install_gauss_mysql_dialect_compatibility(driver)
if backend == "sqlite":
import os
@@ -474,48 +578,98 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
await conn.run_sync(Base.metadata.create_all)
if gauss_mysql:
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
# Schema bootstrap is deliberately best-effort. A single incompatible or
# permission-restricted table must not make the whole Gateway unavailable:
# routes backed by healthy existing tables can still work, while a route
# whose table is missing will report its own request-time error. Keep the
# auto-create-database retries, but contain failures from both the initial
# create_all and the retry path.
schema_bootstrap_lock = None
if gauss_mysql:
try:
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
except Exception:
# A lock-file permission problem is operationally important, but
# the global startup invariant still requires the Gateway to come
# up and report schema errors at the affected routes.
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
await _create_all_with_product_integrity(conn)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database.
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
else:
raise
except Exception:
logger.exception(
"Automatic ORM schema creation failed; continuing Gateway startup "
"with the existing database schema"
)
finally:
try:
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
except Exception:
logger.exception("Could not release openGauss schema bootstrap lock")
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
except Exception:
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
if gauss_mysql:
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
# aborting the transaction with error 322. The native-catalog helper
# above is the only schema repair path for this product.
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
else:
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.