feat: 排行榜日快照分析 + 数据库连接池/索引调优

- 排行榜改为按天预计算快照(admin_leaderboard_daily_stats),请求只读取合并,重统计移至后台调度;过去日不可变、当天 TTL 刷新,避免在线聚合长时间占用数据库
- 为 runs/tool_call_metrics/llm_call_metrics/threads_meta 增加面向时间段+用户聚合的复合索引(model __table_args__ + migration 20260608_01)
- 连接池可调:DatabaseConfig 新增 max_overflow / pool_timeout,接入 postgres/mysql 引擎;config.yaml 上调 pool_size=20/max_overflow=20
- 新增手动执行的索引脚本 scripts/sql/leaderboard_indexes_mysql.sql(幂等,适用于已存在数据、create_all 不补索引且 alembic 关闭的部署)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
AdministratorandClaude Opus 4.8 committed 2026-06-06 00:52:24 +08:00
1 parent 1b75887615
commit e7c745a512
26 files changed
+1980 -564

No files matched your search

@@ -0,0 +1,85 @@
"""Create admin leaderboard daily snapshots and analytics indexes.
Revision ID: 20260608_01
Revises: 20260607_02
Create Date: 2026-06-08
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
from deerflow.persistence.types import PortableJSON
revision = "20260608_01"
down_revision = "20260607_02"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def _create_index(table_name: str, index_name: str, columns: list[str]) -> None:
if _has_table(table_name) and not _has_index(table_name, index_name):
op.create_index(index_name, table_name, columns)
def _drop_index(table_name: str, index_name: str) -> None:
if _has_table(table_name) and _has_index(table_name, index_name):
op.drop_index(index_name, table_name=table_name)
def upgrade() -> None:
if not _has_table("admin_leaderboard_daily_stats"):
op.create_table(
"admin_leaderboard_daily_stats",
sa.Column("stat_date", sa.String(10), nullable=False),
sa.Column("settings_hash", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False, server_default="ready"),
sa.Column("payload_json", PortableJSON(), nullable=True),
sa.Column("message", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("stat_date", "settings_hash", name="pk_admin_leaderboard_daily_stats"),
)
op.create_index("ix_admin_leaderboard_daily_stats_status", "admin_leaderboard_daily_stats", ["status"])
op.create_index("ix_admin_leaderboard_daily_stats_updated_at", "admin_leaderboard_daily_stats", ["updated_at"])
_create_index("runs", "ix_runs_created_user_status", ["created_at", "user_id", "status"])
_create_index("runs", "ix_runs_user_created", ["user_id", "created_at"])
_create_index("runs", "ix_runs_thread_created", ["thread_id", "created_at"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_created_user", ["created_at", "user_id"])
_create_index("tool_call_metrics", "ix_tool_call_metrics_run_created", ["run_id", "created_at"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_created_user", ["created_at", "user_id"])
_create_index("llm_call_metrics", "ix_llm_call_metrics_run_created", ["run_id", "created_at"])
_create_index("threads_meta", "ix_threads_meta_created_user", ["created_at", "user_id"])
_create_index("threads_meta", "ix_threads_meta_user_created", ["user_id", "created_at"])
def downgrade() -> None:
_drop_index("threads_meta", "ix_threads_meta_user_created")
_drop_index("threads_meta", "ix_threads_meta_created_user")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_run_created")
_drop_index("llm_call_metrics", "ix_llm_call_metrics_created_user")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_run_created")
_drop_index("tool_call_metrics", "ix_tool_call_metrics_created_user")
_drop_index("runs", "ix_runs_thread_created")
_drop_index("runs", "ix_runs_user_created")
_drop_index("runs", "ix_runs_created_user_status")
if _has_table("admin_leaderboard_daily_stats"):
op.drop_table("admin_leaderboard_daily_stats")