feat: 排行榜日快照分析 + 数据库连接池/索引调优
- 排行榜改为按天预计算快照(admin_leaderboard_daily_stats),请求只读取合并,重统计移至后台调度;过去日不可变、当天 TTL 刷新,避免在线聚合长时间占用数据库 - 为 runs/tool_call_metrics/llm_call_metrics/threads_meta 增加面向时间段+用户聚合的复合索引(model __table_args__ + migration 20260608_01) - 连接池可调:DatabaseConfig 新增 max_overflow / pool_timeout,接入 postgres/mysql 引擎;config.yaml 上调 pool_size=20/max_overflow=20 - 新增手动执行的索引脚本 scripts/sql/leaderboard_indexes_mysql.sql(幂等,适用于已存在数据、create_all 不补索引且 alembic 关闭的部署) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1b75887615
commit
e7c745a512
26 files changed
+1980
-564
No files matched your search
@@ -11,16 +11,16 @@ None and fall back to in-memory implementations.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
|
||||
def _json_serializer(obj: object) -> str:
|
||||
@@ -133,7 +133,7 @@ async def _run_alembic_upgrade(url: str) -> None:
|
||||
logger.exception("Auto-applying Alembic migrations failed; continuing with create_all schema")
|
||||
|
||||
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
async def _auto_create_postgres_db(url: str) -> None:
|
||||
"""Connect to the ``postgres`` maintenance DB and CREATE DATABASE.
|
||||
|
||||
The target database name is extracted from *url*. The connection is
|
||||
@@ -156,55 +156,57 @@ async def _auto_create_postgres_db(url: str) -> None:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
|
||||
logger.info("Auto-created PostgreSQL database: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def _auto_create_mysql_db(url: str) -> None:
|
||||
"""Create the target MySQL database when it is missing.
|
||||
|
||||
``create_all`` can create tables, but MySQL requires the database/schema to
|
||||
exist before the application URL can connect. We connect to the built-in
|
||||
``mysql`` maintenance database on the same server and issue a conservative
|
||||
``CREATE DATABASE IF NOT EXISTS``.
|
||||
"""
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
|
||||
|
||||
maint_url = parsed.set(database="mysql")
|
||||
maint_engine = create_async_engine(
|
||||
maint_url.render_as_string(hide_password=False),
|
||||
isolation_level="AUTOCOMMIT",
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
quoted = "`" + db_name.replace("`", "``") + "`"
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
f"CREATE DATABASE IF NOT EXISTS {quoted} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||||
)
|
||||
)
|
||||
logger.info("Auto-created MySQL database if missing: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
async def init_engine(
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
async def _auto_create_mysql_db(url: str) -> None:
|
||||
"""Create the target MySQL database when it is missing.
|
||||
|
||||
``create_all`` can create tables, but MySQL requires the database/schema to
|
||||
exist before the application URL can connect. We connect to the built-in
|
||||
``mysql`` maintenance database on the same server and issue a conservative
|
||||
``CREATE DATABASE IF NOT EXISTS``.
|
||||
"""
|
||||
|
||||
parsed = make_url(url)
|
||||
db_name = parsed.database
|
||||
if not db_name:
|
||||
raise ValueError("Cannot auto-create MySQL database: no database name in URL")
|
||||
|
||||
maint_url = parsed.set(database="mysql")
|
||||
maint_engine = create_async_engine(
|
||||
maint_url.render_as_string(hide_password=False),
|
||||
isolation_level="AUTOCOMMIT",
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
quoted = "`" + db_name.replace("`", "``") + "`"
|
||||
try:
|
||||
async with maint_engine.connect() as conn:
|
||||
await conn.execute(
|
||||
text(
|
||||
f"CREATE DATABASE IF NOT EXISTS {quoted} "
|
||||
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||||
)
|
||||
)
|
||||
logger.info("Auto-created MySQL database if missing: %s", db_name)
|
||||
finally:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
url: str = "",
|
||||
echo: bool = False,
|
||||
pool_size: int = 5,
|
||||
max_overflow: int = 10,
|
||||
pool_timeout: int = 30,
|
||||
sqlite_dir: str = "",
|
||||
) -> None:
|
||||
"""Create the async engine and session factory, then auto-create tables.
|
||||
@@ -213,7 +215,9 @@ async def init_engine(
|
||||
backend: "memory", "sqlite", "postgres", or "mysql".
|
||||
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
|
||||
echo: Echo SQL to log.
|
||||
pool_size: Postgres connection pool size.
|
||||
pool_size: Base connection pool size (postgres/mysql).
|
||||
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
|
||||
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
|
||||
sqlite_dir: Directory to create for SQLite (ensured to exist).
|
||||
"""
|
||||
global _engine, _session_factory
|
||||
@@ -271,6 +275,8 @@ async def init_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
@@ -279,6 +285,8 @@ async def init_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
@@ -302,37 +310,39 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend == "postgres" and "does not exist" in str(exc):
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, pool_pre_ping=True, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=1800,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
@@ -351,6 +361,8 @@ async def init_engine_from_config(config) -> None:
|
||||
url=config.app_sqlalchemy_url,
|
||||
echo=config.echo_sql,
|
||||
pool_size=config.pool_size,
|
||||
max_overflow=getattr(config, "max_overflow", 10),
|
||||
pool_timeout=getattr(config, "pool_timeout", 30),
|
||||
sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "",
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user