sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
@@ -215,7 +215,7 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
@@ -226,8 +226,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Table-level ``UniqueConstraint`` objects
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
@@ -261,8 +261,7 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
@@ -271,24 +270,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
@@ -332,11 +328,67 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
@@ -394,6 +446,10 @@ async def init_engine(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -474,48 +530,70 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
# Schema bootstrap is deliberately best-effort. A single incompatible or
|
||||
# permission-restricted table must not make the whole Gateway unavailable:
|
||||
# routes backed by healthy existing tables can still work, while a route
|
||||
# whose table is missing will report its own request-time error. Keep the
|
||||
# auto-create-database retries, but contain failures from both the initial
|
||||
# create_all and the retry path.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database.
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Automatic ORM schema creation failed; continuing Gateway startup "
|
||||
"with the existing database schema"
|
||||
)
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
|
||||
Reference in new issue
Block a user