sync snapshot from LAN repo (source codex/kb-context-retrieval-20261021 @ b69568e4)
This commit is contained in:
1 parent
ea9dcff9c0
commit
4cf657f725
781 files changed
+13559
-104852
No files matched your search
+227
-418
@@ -11,20 +11,14 @@ None and fall back to in-memory implementations.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import weakref
|
||||
from contextvars import ContextVar, Token
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
@@ -44,167 +38,21 @@ _session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
# "got Future attached to a different loop". We capture the engine's home loop here so
|
||||
# best-effort writes coming from worker threads / other loops can be routed back to it
|
||||
# (see ``fire_and_forget_db_write``).
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
_performance_request_context: ContextVar[tuple[str, Any] | None] = ContextVar("deerflow_performance_request", default=None)
|
||||
|
||||
|
||||
def bind_performance_request(request_id: str) -> Token:
|
||||
"""Bind a request ID to this task so slow SQL logs can be correlated safely."""
|
||||
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return _performance_request_context.set((request_id, weakref.ref(task) if task is not None else None))
|
||||
|
||||
|
||||
def reset_performance_request(token: Token) -> None:
|
||||
_performance_request_context.reset(token)
|
||||
|
||||
|
||||
def _current_performance_request_id() -> str:
|
||||
context = _performance_request_context.get()
|
||||
if context is None:
|
||||
return "-"
|
||||
try:
|
||||
task = asyncio.current_task()
|
||||
except RuntimeError:
|
||||
task = None
|
||||
return context[0] if task is not None and context[1] is not None and context[1]() is task else "-"
|
||||
|
||||
|
||||
def _slow_sql_threshold_ms() -> float:
|
||||
try:
|
||||
return max(0.0, min(float(os.getenv("DEERFLOW_SLOW_SQL_LOG_MS", "500")), 60_000.0))
|
||||
except (TypeError, ValueError):
|
||||
return 500.0
|
||||
|
||||
|
||||
def _sql_operation(statement: str) -> str:
|
||||
parts = statement.lstrip().split(None, 1)
|
||||
candidate = parts[0].upper() if parts else "EMPTY"
|
||||
return candidate if re.fullmatch(r"[A-Z_]+", candidate) else "OTHER"
|
||||
|
||||
|
||||
def _sql_table(statement: str) -> str:
|
||||
"""Return the target/source table only, without query text or values."""
|
||||
match = re.search(
|
||||
r"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+([`\"\w.]+)",
|
||||
statement,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if match is None:
|
||||
match = re.search(r"\bFROM\s+([`\"\w.]+)", statement, re.IGNORECASE)
|
||||
if match is None:
|
||||
return "unknown"
|
||||
return match.group(1).strip("`\".")[-128:] or "unknown"
|
||||
|
||||
|
||||
def _sql_fingerprint(statement: str) -> str:
|
||||
return hashlib.sha256(statement.encode("utf-8", "replace")).hexdigest()[:12]
|
||||
|
||||
|
||||
def _database_error_metadata(exception: BaseException | None) -> tuple[str, str, str]:
|
||||
"""Extract safe driver diagnostics without logging SQL or bound values."""
|
||||
if exception is None:
|
||||
return "unknown", "-", "-"
|
||||
|
||||
code = getattr(exception, "errno", None) or getattr(exception, "code", None)
|
||||
args = getattr(exception, "args", ())
|
||||
if code is None and args and isinstance(args[0], (int, str)):
|
||||
candidate = str(args[0])
|
||||
if candidate.isdigit():
|
||||
code = candidate
|
||||
|
||||
sqlstate = getattr(exception, "sqlstate", None) or getattr(exception, "pgcode", None)
|
||||
diagnostic = getattr(exception, "diag", None)
|
||||
constraint = getattr(diagnostic, "constraint_name", None)
|
||||
|
||||
def _safe_identifier(value: Any) -> str:
|
||||
text_value = str(value or "")
|
||||
return text_value[:128] if re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", text_value) else "-"
|
||||
|
||||
return _safe_identifier(code), _safe_identifier(sqlstate), _safe_identifier(constraint)
|
||||
|
||||
|
||||
def _install_slow_sql_logging(engine: AsyncEngine, backend: str) -> None:
|
||||
"""Log slow SQL timings without logging query text or bound values."""
|
||||
|
||||
sync_engine = engine.sync_engine
|
||||
|
||||
def before_cursor_execute(_conn, _cursor, statement, _parameters, context, _executemany):
|
||||
if context is not None:
|
||||
context._deerflow_perf_started = time.perf_counter()
|
||||
context._deerflow_perf_operation = _sql_operation(statement)
|
||||
context._deerflow_perf_statement = statement
|
||||
|
||||
def after_cursor_execute(_conn, _cursor, _statement, _parameters, context, executemany):
|
||||
if context is None:
|
||||
return
|
||||
started = getattr(context, "_deerflow_perf_started", None)
|
||||
if started is None:
|
||||
return
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
if elapsed_ms >= _slow_sql_threshold_ms():
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", ""))
|
||||
logger.warning(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%.2f executemany=%s fingerprint=%s outcome=ok",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER"),
|
||||
_sql_table(getattr(context, "_deerflow_perf_statement", "")),
|
||||
elapsed_ms,
|
||||
bool(executemany),
|
||||
fingerprint,
|
||||
)
|
||||
|
||||
def handle_error(exception_context):
|
||||
context = exception_context.execution_context
|
||||
started = getattr(context, "_deerflow_perf_started", None) if context is not None else None
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 2) if started is not None else None
|
||||
fingerprint = _sql_fingerprint(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
original_exception = exception_context.original_exception
|
||||
error_code, sqlstate, constraint = _database_error_metadata(original_exception)
|
||||
table = _sql_table(getattr(context, "_deerflow_perf_statement", "")) if context is not None else "unknown"
|
||||
duplicate_key = error_code == "1062" or sqlstate == "23505" or (
|
||||
table == "scheduled_task_runs"
|
||||
and "UNIQUE constraint failed" in str(original_exception or "")
|
||||
)
|
||||
log = logger.info if table == "scheduled_task_runs" and duplicate_key else logger.error
|
||||
outcome = "duplicate_ignored" if table == "scheduled_task_runs" and duplicate_key else "error"
|
||||
log(
|
||||
"PERF db_query request_id=%s backend=%s operation=%s table=%s duration_ms=%s executemany=%s fingerprint=%s outcome=%s error_type=%s error_code=%s sqlstate=%s constraint=%s",
|
||||
_current_performance_request_id(),
|
||||
backend,
|
||||
getattr(context, "_deerflow_perf_operation", "OTHER") if context is not None else "OTHER",
|
||||
table,
|
||||
elapsed_ms,
|
||||
bool(getattr(context, "executemany", False)) if context is not None else False,
|
||||
fingerprint,
|
||||
outcome,
|
||||
type(original_exception).__name__ if original_exception is not None else "unknown",
|
||||
error_code,
|
||||
sqlstate,
|
||||
constraint,
|
||||
)
|
||||
|
||||
event.listen(sync_engine, "before_cursor_execute", before_cursor_execute)
|
||||
event.listen(sync_engine, "after_cursor_execute", after_cursor_execute)
|
||||
event.listen(sync_engine, "handle_error", handle_error)
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
_engine_loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
# These are the complete workflow-studio persistence tables owned by DeerFlow.
|
||||
# Keep this explicit allow-list instead of treating ``workflow_%`` as a SQL
|
||||
# wildcard: another application might share a schema and have its own table
|
||||
# with that prefix.
|
||||
_WORKFLOW_SCHEMA_TABLE_NAMES = (
|
||||
"workflow_definitions",
|
||||
"workflow_versions",
|
||||
"workflow_runs",
|
||||
"workflow_node_runs",
|
||||
"workflow_run_events",
|
||||
"workflow_run_artifacts",
|
||||
"workflow_data_sources",
|
||||
)
|
||||
|
||||
|
||||
def _add_runtime_vendor_path() -> None:
|
||||
@@ -233,42 +81,6 @@ def _mysql_driver_name(url: str) -> str:
|
||||
return "asyncmy"
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
def _pooled_engine_kwargs(*, backend, url, echo, pool_size, max_overflow, pool_timeout, pool_recycle, command_timeout):
|
||||
"""Use identical pool/deadline settings on initial connect and DB-create retry."""
|
||||
kwargs = dict(
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
# asyncpg command deadlines also bound the SELECT used by pool_pre_ping.
|
||||
# These are not MySQL/psycopg connection arguments.
|
||||
if backend in {"postgres", "gauss"} and make_url(url).drivername.endswith("+asyncpg"):
|
||||
kwargs["connect_args"] = {"command_timeout": command_timeout}
|
||||
return kwargs
|
||||
|
||||
|
||||
async def _run_alembic_upgrade(url: str) -> None:
|
||||
"""Apply pending Alembic migrations against ``url`` (``alembic upgrade head``).
|
||||
|
||||
@@ -364,7 +176,7 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
@@ -403,7 +215,7 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
@@ -414,14 +226,14 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
|
||||
on an existing database, weakening dedupe.
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
|
||||
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
|
||||
``BeijingDateTime``) resolve to the right backend type. Everything is
|
||||
per-column best-effort: a failure is logged and startup continues.
|
||||
@@ -446,8 +258,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
continue
|
||||
existing_cols = {c["name"] for c in inspector.get_columns(table.name)}
|
||||
missing = [c for c in table.columns if c.name not in existing_cols]
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
for col in missing:
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
@@ -456,21 +270,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if not idx.name or idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
@@ -514,67 +328,67 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
@@ -627,32 +441,24 @@ async def init_engine(
|
||||
pool_size: int = 5,
|
||||
max_overflow: int = 10,
|
||||
pool_timeout: int = 30,
|
||||
pool_recycle: int | None = None,
|
||||
command_timeout: float | None = 30,
|
||||
pool_recycle: int = 1800,
|
||||
sqlite_dir: str = "",
|
||||
) -> None:
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
|
||||
Args:
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
pool_size: Base connection pool size (postgres/gauss/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
|
||||
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
|
||||
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
|
||||
command_timeout: asyncpg command deadline, including pool pre-ping (None disables).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory, _engine_loop
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if pool_recycle is None:
|
||||
pool_recycle = 300 if backend == "postgres" else 1800
|
||||
|
||||
if backend == "memory":
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
@@ -669,25 +475,25 @@ async def init_engine(
|
||||
except RuntimeError:
|
||||
_engine_loop = None
|
||||
|
||||
if backend in {"postgres", "gauss"}:
|
||||
if backend in {"postgres", "gauss"}:
|
||||
try:
|
||||
import asyncpg # noqa: F401
|
||||
except ImportError:
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
extra = "gauss" if backend == "gauss" else "postgres"
|
||||
raise ImportError(
|
||||
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
|
||||
f"Install it with:\n uv sync --extra {extra}\n"
|
||||
"Or switch to backend: sqlite in config.yaml for single-node deployment."
|
||||
) from None
|
||||
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
|
||||
raise ImportError(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -703,6 +509,10 @@ async def init_engine(
|
||||
install_gauss_mysql_dialect_compatibility(driver)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
|
||||
from sqlalchemy import event
|
||||
|
||||
os.makedirs(sqlite_dir or ".", exist_ok=True)
|
||||
_engine = create_async_engine(url, echo=echo, json_serializer=_json_serializer)
|
||||
|
||||
@@ -727,28 +537,31 @@ async def init_engine(
|
||||
cursor.execute("PRAGMA foreign_keys=ON;")
|
||||
finally:
|
||||
cursor.close()
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
elif backend in {"postgres", "gauss"}:
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
elif backend == "mysql":
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown persistence backend: {backend!r}")
|
||||
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Auto-create tables for fresh deployments, then let Alembic apply any
|
||||
@@ -799,13 +612,14 @@ async def init_engine(
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
@@ -816,13 +630,14 @@ async def init_engine(
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
**_pooled_engine_kwargs(
|
||||
backend=backend, url=url, echo=echo, pool_size=pool_size,
|
||||
max_overflow=max_overflow, pool_timeout=pool_timeout,
|
||||
pool_recycle=pool_recycle, command_timeout=command_timeout,
|
||||
),
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_install_slow_sql_logging(_engine, backend)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
@@ -860,7 +675,7 @@ async def init_engine(
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
# await _run_alembic_upgrade(url)
|
||||
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
|
||||
|
||||
|
||||
async def init_engine_from_config(config) -> None:
|
||||
@@ -875,17 +690,14 @@ async def init_engine_from_config(config) -> None:
|
||||
pool_size=config.pool_size,
|
||||
max_overflow=getattr(config, "max_overflow", 10),
|
||||
pool_timeout=getattr(config, "pool_timeout", 30),
|
||||
pool_recycle=getattr(config, "pool_recycle", None),
|
||||
command_timeout=getattr(config, "command_timeout", 30),
|
||||
pool_recycle=getattr(config, "pool_recycle", 1800),
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession] | None:
|
||||
"""Return the async session factory, or None if backend=memory."""
|
||||
from deerflow.persistence.loop_isolation import isolated_session_factory
|
||||
|
||||
return isolated_session_factory() or _session_factory
|
||||
return _session_factory
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine | None:
|
||||
@@ -893,73 +705,73 @@ def get_engine() -> AsyncEngine | None:
|
||||
return _engine
|
||||
|
||||
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
def get_engine_loop() -> asyncio.AbstractEventLoop | None:
|
||||
"""Return the event loop that owns the engine's connection pool (or None)."""
|
||||
return _engine_loop
|
||||
|
||||
|
||||
async def rebuild_workflow_schema_for_mysql() -> list[str]:
|
||||
"""Drop and recreate DeerFlow's workflow tables in the current MySQL schema.
|
||||
|
||||
This is a deliberately narrow recovery operation for an unused Workflow
|
||||
Studio installation. It never uses a prefix wildcard, never changes the
|
||||
selected database, and refuses to run unless the engine is connected to
|
||||
MySQL and ``SELECT DATABASE()`` confirms the URL's exact target schema.
|
||||
|
||||
Returns the allow-listed workflow tables that existed before the rebuild.
|
||||
"""
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: persistence engine is not initialized")
|
||||
if engine.dialect.name != "mysql":
|
||||
raise RuntimeError(
|
||||
"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA is supported only for a MySQL persistence backend"
|
||||
)
|
||||
|
||||
expected_database = engine.url.database
|
||||
if not expected_database:
|
||||
raise RuntimeError("Cannot rebuild workflow schema: MySQL URL has no database name")
|
||||
|
||||
# Make the operation safe when this helper is invoked independently from
|
||||
# normal app startup as well.
|
||||
import deerflow.persistence.models # noqa: F401 — register ORM models
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
missing_models = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name not in Base.metadata.tables]
|
||||
if missing_models:
|
||||
raise RuntimeError(
|
||||
"Cannot rebuild workflow schema: ORM metadata is missing " + ", ".join(missing_models)
|
||||
)
|
||||
workflow_tables = [Base.metadata.tables[name] for name in _WORKFLOW_SCHEMA_TABLE_NAMES]
|
||||
|
||||
def _rebuild(sync_conn) -> list[str]:
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
current_database = sync_conn.exec_driver_sql("SELECT DATABASE()").scalar_one_or_none()
|
||||
if current_database != expected_database:
|
||||
raise RuntimeError(
|
||||
"Refusing workflow schema rebuild: connected database "
|
||||
f"{current_database!r} does not match configured database {expected_database!r}"
|
||||
)
|
||||
|
||||
existing = set(sa_inspect(sync_conn).get_table_names())
|
||||
present = [name for name in _WORKFLOW_SCHEMA_TABLE_NAMES if name in existing]
|
||||
# SQLAlchemy orders parent/child tables around the foreign key from
|
||||
# workflow_versions to workflow_definitions. Only tables in the exact
|
||||
# allow-list above are ever passed to drop_all/create_all.
|
||||
Base.metadata.drop_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
Base.metadata.create_all(sync_conn, tables=workflow_tables, checkfirst=True)
|
||||
return present
|
||||
|
||||
async with engine.begin() as conn:
|
||||
rebuilt = await conn.run_sync(_rebuild)
|
||||
|
||||
logger.warning(
|
||||
"Rebuilt workflow schema in current MySQL database %s; tables reset: %s",
|
||||
expected_database,
|
||||
", ".join(rebuilt) if rebuilt else "none (created fresh)",
|
||||
)
|
||||
return rebuilt
|
||||
|
||||
|
||||
def fire_and_forget_db_write(coro) -> None:
|
||||
@@ -1008,13 +820,10 @@ def _run_coro_on_throwaway_loop(coro):
|
||||
|
||||
Only reached when there is no usable engine home loop, or when the caller is
|
||||
*synchronously* inside the home loop (routing back would deadlock). This
|
||||
uses an isolated, non-pooled engine for compatibility reads. The coroutine
|
||||
must obtain its session factory inside its async body so the isolated
|
||||
context applies; no connection is returned to the Gateway pool.
|
||||
touches the shared connection pool from a throwaway loop, so it can still leak
|
||||
a dead-loop-bound connection — but the common worker-thread path no longer
|
||||
gets here (see ``run_db_blocking``).
|
||||
"""
|
||||
from deerflow.persistence.loop_isolation import run_isolated_db_read
|
||||
|
||||
coro = run_isolated_db_read(coro)
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
in_loop = True
|
||||
|
||||
Reference in new issue
Block a user