feat: 排行榜日快照分析 + 数据库连接池/索引调优
- 排行榜改为按天预计算快照(admin_leaderboard_daily_stats),请求只读取合并,重统计移至后台调度;过去日不可变、当天 TTL 刷新,避免在线聚合长时间占用数据库 - 为 runs/tool_call_metrics/llm_call_metrics/threads_meta 增加面向时间段+用户聚合的复合索引(model __table_args__ + migration 20260608_01) - 连接池可调:DatabaseConfig 新增 max_overflow / pool_timeout,接入 postgres/mysql 引擎;config.yaml 上调 pool_size=20/max_overflow=20 - 新增手动执行的索引脚本 scripts/sql/leaderboard_indexes_mysql.sql(幂等,适用于已存在数据、create_all 不补索引且 alembic 关闭的部署) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1b75887615
commit
e7c745a512
26 files changed
+1980
-564
No files matched your search
+85
@@ -0,0 +1,85 @@
|
||||
"""Create admin leaderboard daily snapshots and analytics indexes.
|
||||
|
||||
Revision ID: 20260608_01
|
||||
Revises: 20260607_02
|
||||
Create Date: 2026-06-08
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
|
||||
revision = "20260608_01"
|
||||
down_revision = "20260607_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def _create_index(table_name: str, index_name: str, columns: list[str]) -> None:
|
||||
if _has_table(table_name) and not _has_index(table_name, index_name):
|
||||
op.create_index(index_name, table_name, columns)
|
||||
|
||||
|
||||
def _drop_index(table_name: str, index_name: str) -> None:
|
||||
if _has_table(table_name) and _has_index(table_name, index_name):
|
||||
op.drop_index(index_name, table_name=table_name)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table("admin_leaderboard_daily_stats"):
|
||||
op.create_table(
|
||||
"admin_leaderboard_daily_stats",
|
||||
sa.Column("stat_date", sa.String(10), nullable=False),
|
||||
sa.Column("settings_hash", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="ready"),
|
||||
sa.Column("payload_json", PortableJSON(), nullable=True),
|
||||
sa.Column("message", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("generated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint("stat_date", "settings_hash", name="pk_admin_leaderboard_daily_stats"),
|
||||
)
|
||||
op.create_index("ix_admin_leaderboard_daily_stats_status", "admin_leaderboard_daily_stats", ["status"])
|
||||
op.create_index("ix_admin_leaderboard_daily_stats_updated_at", "admin_leaderboard_daily_stats", ["updated_at"])
|
||||
|
||||
_create_index("runs", "ix_runs_created_user_status", ["created_at", "user_id", "status"])
|
||||
_create_index("runs", "ix_runs_user_created", ["user_id", "created_at"])
|
||||
_create_index("runs", "ix_runs_thread_created", ["thread_id", "created_at"])
|
||||
_create_index("tool_call_metrics", "ix_tool_call_metrics_created_user", ["created_at", "user_id"])
|
||||
_create_index("tool_call_metrics", "ix_tool_call_metrics_run_created", ["run_id", "created_at"])
|
||||
_create_index("llm_call_metrics", "ix_llm_call_metrics_created_user", ["created_at", "user_id"])
|
||||
_create_index("llm_call_metrics", "ix_llm_call_metrics_run_created", ["run_id", "created_at"])
|
||||
_create_index("threads_meta", "ix_threads_meta_created_user", ["created_at", "user_id"])
|
||||
_create_index("threads_meta", "ix_threads_meta_user_created", ["user_id", "created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
_drop_index("threads_meta", "ix_threads_meta_user_created")
|
||||
_drop_index("threads_meta", "ix_threads_meta_created_user")
|
||||
_drop_index("llm_call_metrics", "ix_llm_call_metrics_run_created")
|
||||
_drop_index("llm_call_metrics", "ix_llm_call_metrics_created_user")
|
||||
_drop_index("tool_call_metrics", "ix_tool_call_metrics_run_created")
|
||||
_drop_index("tool_call_metrics", "ix_tool_call_metrics_created_user")
|
||||
_drop_index("runs", "ix_runs_thread_created")
|
||||
_drop_index("runs", "ix_runs_user_created")
|
||||
_drop_index("runs", "ix_runs_created_user_status")
|
||||
if _has_table("admin_leaderboard_daily_stats"):
|
||||
op.drop_table("admin_leaderboard_daily_stats")
|
||||
Reference in new issue
Block a user