feat: 排行榜日快照分析 + 数据库连接池/索引调优
- 排行榜改为按天预计算快照(admin_leaderboard_daily_stats),请求只读取合并,重统计移至后台调度;过去日不可变、当天 TTL 刷新,避免在线聚合长时间占用数据库 - 为 runs/tool_call_metrics/llm_call_metrics/threads_meta 增加面向时间段+用户聚合的复合索引(model __table_args__ + migration 20260608_01) - 连接池可调:DatabaseConfig 新增 max_overflow / pool_timeout,接入 postgres/mysql 引擎;config.yaml 上调 pool_size=20/max_overflow=20 - 新增手动执行的索引脚本 scripts/sql/leaderboard_indexes_mysql.sql(幂等,适用于已存在数据、create_all 不补索引且 alembic 关闭的部署) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1b75887615
commit
e7c745a512
26 files changed
+1980
-564
No files matched your search
+61
-47
@@ -11,13 +11,13 @@ blocking, making a unified file safe for both workloads. Writers
|
||||
that contend for the lock wait via the default 5-second sqlite3
|
||||
busy timeout rather than failing immediately.
|
||||
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
Postgres mode: both use the same database URL but maintain independent
|
||||
connection pools with different lifecycles.
|
||||
|
||||
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
|
||||
not have a built-in MySQL backend in this project, so deployments that choose
|
||||
MySQL for business tables should configure the legacy ``checkpointer:``
|
||||
section separately (usually sqlite) to preserve conversation checkpoints.
|
||||
|
||||
Memory mode: checkpointer uses MemorySaver, app uses in-memory stores.
|
||||
No database is initialized.
|
||||
@@ -26,12 +26,12 @@ Sensitive values (postgres_url) should use $VAR syntax in config.yaml
|
||||
to reference environment variables from .env:
|
||||
|
||||
database:
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
backend: postgres
|
||||
postgres_url: $DATABASE_URL
|
||||
|
||||
The $VAR resolution is handled by AppConfig.resolve_env_variables()
|
||||
before this config is instantiated -- DatabaseConfig itself does not
|
||||
need to do any environment variable processing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -42,40 +42,54 @@ from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
)
|
||||
class DatabaseConfig(BaseModel):
|
||||
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
|
||||
default="memory",
|
||||
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
|
||||
)
|
||||
sqlite_dir: str = Field(
|
||||
default=".deer-flow/data",
|
||||
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
|
||||
)
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
postgres_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"PostgreSQL connection URL, shared by checkpointer and app. "
|
||||
"Use $DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: postgresql://user:pass@host:5432/deerflow "
|
||||
"(the +asyncpg driver suffix is added automatically where needed)."
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"MySQL connection URL for the application ORM. "
|
||||
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
|
||||
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
mysql_url: str = Field(
|
||||
default="",
|
||||
description=(
|
||||
"MySQL connection URL for the application ORM. "
|
||||
"Use $MYSQL_DATABASE_URL in config.yaml to reference .env. "
|
||||
"Example: mysql+asyncmy://user:pass@host:3306/deerflow?charset=utf8mb4. "
|
||||
"If mysql:// is provided, the +asyncmy driver suffix is added automatically."
|
||||
),
|
||||
)
|
||||
echo_sql: bool = Field(
|
||||
default=False,
|
||||
description="Echo all SQL statements to log (debug only).",
|
||||
)
|
||||
pool_size: int = Field(
|
||||
default=5,
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
description="Connection pool size for the app ORM engine (postgres/mysql only).",
|
||||
)
|
||||
max_overflow: int = Field(
|
||||
default=10,
|
||||
description=(
|
||||
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
|
||||
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
|
||||
),
|
||||
)
|
||||
pool_timeout: int = Field(
|
||||
default=30,
|
||||
description=(
|
||||
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
|
||||
"SQLAlchemy default is 30."
|
||||
),
|
||||
)
|
||||
|
||||
# -- Derived helpers (not user-configured) --
|
||||
@@ -108,16 +122,16 @@ class DatabaseConfig(BaseModel):
|
||||
"""SQLAlchemy async URL for the application ORM engine."""
|
||||
if self.backend == "sqlite":
|
||||
return f"sqlite+aiosqlite:///{self.sqlite_path}"
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
if url.startswith("mysql://"):
|
||||
url = url.replace("mysql://", "mysql+asyncmy://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
if self.backend == "postgres":
|
||||
url = self.postgres_url
|
||||
if url.startswith("postgresql://"):
|
||||
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||
return url
|
||||
if self.backend == "mysql":
|
||||
url = self.mysql_url
|
||||
if not url:
|
||||
raise ValueError("database.mysql_url is required for backend='mysql'")
|
||||
if url.startswith("mysql://"):
|
||||
url = url.replace("mysql://", "mysql+asyncmy://", 1)
|
||||
return url
|
||||
raise ValueError(f"No SQLAlchemy URL for backend={self.backend!r}")
|
||||
Reference in new issue
Block a user