feat: add admin user leaderboard analytics

This commit is contained in:
Administrator committed 2026-06-04 19:35:11 +08:00
1 parent 1e4ef070cf
commit 8f80b7c1ef
7 files changed
+1060 -1

No files matched your search

@@ -15,8 +15,11 @@ from __future__ import annotations
import logging
from datetime import datetime, timedelta
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from sqlalchemy import and_, case, desc, func, select
from app.gateway.deps import (
get_agent_store,
@@ -28,6 +31,11 @@ from deerflow.agents.memory import BuiltinFileProvider
from deerflow.config.paths import get_paths
from deerflow.runtime import serialize_channel_values
from app.gateway.routers.memory import _build_v1_provider, _is_v1
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.run.model import RunRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
@@ -45,6 +53,429 @@ async def _require_admin(request: Request) -> None:
raise HTTPException(status_code=403, detail="Admin only")
# ── Leaderboard / analytics ─────────────────────────────────────────────────
class AdminLeaderboardUser(BaseModel):
user_id: str
email: str = ""
system_role: str = "user"
thread_count: int = 0
run_count: int = 0
qa_count: int = 0
llm_call_count: int = 0
tool_call_count: int = 0
skill_call_count: int = 0
total_tokens: int = 0
input_tokens: int = 0
output_tokens: int = 0
last_active_at: str = ""
class AdminLeaderboardSkill(BaseModel):
name: str
call_count: int = 0
user_count: int = 0
success_count: int = 0
error_count: int = 0
avg_duration_ms: int = 0
last_used_at: str = ""
class AdminLeaderboardQuestion(BaseModel):
text: str
user_id: str = ""
email: str = ""
thread_id: str = ""
run_id: str = ""
asked_at: str = ""
class AdminLeaderboardTrendPoint(BaseModel):
date: str
runs: int = 0
questions: int = 0
tokens: int = 0
users: int = 0
class AdminLeaderboardOverview(BaseModel):
total_users: int = 0
active_users: int = 0
total_threads: int = 0
total_runs: int = 0
total_questions: int = 0
total_tokens: int = 0
total_skill_calls: int = 0
total_tool_calls: int = 0
error_runs: int = 0
class AdminLeaderboardResponse(BaseModel):
range_days: int
since: str
until: str
overview: AdminLeaderboardOverview
users_by_activity: list[AdminLeaderboardUser] = Field(default_factory=list)
users_by_questions: list[AdminLeaderboardUser] = Field(default_factory=list)
users_by_tokens: list[AdminLeaderboardUser] = Field(default_factory=list)
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
frequent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
class AdminUserAnalyticsResponse(BaseModel):
range_days: int
since: str
until: str
user: AdminLeaderboardUser
overview: AdminLeaderboardOverview
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
def _now() -> datetime:
return datetime.now()
def _iso(value) -> str:
return value.isoformat() if isinstance(value, datetime) else str(value or "")
def _range_start(days: int) -> datetime:
return _now() - timedelta(days=max(1, min(days, 365)))
def _question_count_expr():
return func.sum(case((RunRow.first_human_message.is_not(None), 1), else_=0))
def _run_time_filter(since: datetime, until: datetime):
return and_(RunRow.created_at >= since, RunRow.created_at <= until)
def _tool_time_filter(since: datetime, until: datetime):
return and_(ToolCallMetricRow.created_at >= since, ToolCallMetricRow.created_at <= until)
async def _query_user_leaderboard(session, since: datetime, until: datetime, user_id: str | None = None) -> list[AdminLeaderboardUser]:
run_agg = (
select(
RunRow.user_id.label("user_id"),
func.count(RunRow.run_id).label("run_count"),
_question_count_expr().label("qa_count"),
func.coalesce(func.sum(RunRow.llm_call_count), 0).label("llm_call_count"),
func.coalesce(func.sum(RunRow.total_tokens), 0).label("total_tokens"),
func.coalesce(func.sum(RunRow.total_input_tokens), 0).label("input_tokens"),
func.coalesce(func.sum(RunRow.total_output_tokens), 0).label("output_tokens"),
func.max(RunRow.updated_at).label("last_active_at"),
)
.where(_run_time_filter(since, until))
.group_by(RunRow.user_id)
.subquery()
)
thread_agg = (
select(ThreadMetaRow.user_id.label("user_id"), func.count(ThreadMetaRow.thread_id).label("thread_count"))
.where(and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until))
.group_by(ThreadMetaRow.user_id)
.subquery()
)
tool_agg = (
select(
ToolCallMetricRow.user_id.label("user_id"),
func.count(ToolCallMetricRow.id).label("tool_call_count"),
func.sum(case((ToolCallMetricRow.skill_name.is_not(None), 1), else_=0)).label("skill_call_count"),
)
.where(_tool_time_filter(since, until))
.group_by(ToolCallMetricRow.user_id)
.subquery()
)
stmt = (
select(
UserRow.id,
UserRow.email,
UserRow.system_role,
func.coalesce(thread_agg.c.thread_count, 0),
func.coalesce(run_agg.c.run_count, 0),
func.coalesce(run_agg.c.qa_count, 0),
func.coalesce(run_agg.c.llm_call_count, 0),
func.coalesce(tool_agg.c.tool_call_count, 0),
func.coalesce(tool_agg.c.skill_call_count, 0),
func.coalesce(run_agg.c.total_tokens, 0),
func.coalesce(run_agg.c.input_tokens, 0),
func.coalesce(run_agg.c.output_tokens, 0),
run_agg.c.last_active_at,
)
.outerjoin(run_agg, run_agg.c.user_id == UserRow.id)
.outerjoin(thread_agg, thread_agg.c.user_id == UserRow.id)
.outerjoin(tool_agg, tool_agg.c.user_id == UserRow.id)
)
if user_id:
stmt = stmt.where(UserRow.id == user_id)
stmt = stmt.order_by(
desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)),
desc(func.coalesce(run_agg.c.total_tokens, 0)),
)
rows = (await session.execute(stmt)).all()
return [
AdminLeaderboardUser(
user_id=r[0],
email=r[1] or "",
system_role=r[2] or "user",
thread_count=int(r[3] or 0),
run_count=int(r[4] or 0),
qa_count=int(r[5] or 0),
llm_call_count=int(r[6] or 0),
tool_call_count=int(r[7] or 0),
skill_call_count=int(r[8] or 0),
total_tokens=int(r[9] or 0),
input_tokens=int(r[10] or 0),
output_tokens=int(r[11] or 0),
last_active_at=_iso(r[12]),
)
for r in rows
]
async def _query_call_rankings(session, since: datetime, until: datetime, *, user_id: str | None, field: str, limit: int) -> list[AdminLeaderboardSkill]:
name_col = ToolCallMetricRow.skill_name if field == "skill" else ToolCallMetricRow.tool_name
call_count = func.count(ToolCallMetricRow.id).label("call_count")
last_used_at = func.max(ToolCallMetricRow.created_at).label("last_used_at")
stmt = (
select(
name_col.label("name"),
call_count,
func.count(func.distinct(ToolCallMetricRow.user_id)).label("user_count"),
func.sum(case((ToolCallMetricRow.status == "success", 1), else_=0)).label("success_count"),
func.sum(case((ToolCallMetricRow.status != "success", 1), else_=0)).label("error_count"),
func.coalesce(func.avg(ToolCallMetricRow.duration_ms), 0).label("avg_duration_ms"),
last_used_at,
)
.where(_tool_time_filter(since, until), name_col.is_not(None), name_col != "")
.group_by(name_col)
.order_by(call_count.desc(), last_used_at.desc())
.limit(limit)
)
if user_id:
stmt = stmt.where(ToolCallMetricRow.user_id == user_id)
rows = (await session.execute(stmt)).all()
return [
AdminLeaderboardSkill(
name=r[0] or "",
call_count=int(r[1] or 0),
user_count=int(r[2] or 0),
success_count=int(r[3] or 0),
error_count=int(r[4] or 0),
avg_duration_ms=int(r[5] or 0),
last_used_at=_iso(r[6]),
)
for r in rows
]
async def _query_recent_questions(session, since: datetime, until: datetime, *, user_id: str | None, limit: int) -> list[AdminLeaderboardQuestion]:
stmt = (
select(RunRow.first_human_message, RunRow.user_id, UserRow.email, RunRow.thread_id, RunRow.run_id, RunRow.created_at)
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
.order_by(RunRow.created_at.desc())
.limit(limit)
)
if user_id:
stmt = stmt.where(RunRow.user_id == user_id)
rows = (await session.execute(stmt)).all()
return [
AdminLeaderboardQuestion(
text=(r[0] or "")[:500],
user_id=r[1] or "",
email=r[2] or "",
thread_id=r[3] or "",
run_id=r[4] or "",
asked_at=_iso(r[5]),
)
for r in rows
]
async def _query_frequent_questions(session, since: datetime, until: datetime, limit: int) -> list[AdminLeaderboardQuestion]:
stmt = (
select(
RunRow.first_human_message,
func.count(RunRow.run_id).label("count"),
func.max(RunRow.user_id).label("user_id"),
func.max(UserRow.email).label("email"),
func.max(RunRow.created_at).label("asked_at"),
)
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
.group_by(RunRow.first_human_message)
.having(func.count(RunRow.run_id) > 1)
.order_by(desc("count"), desc("asked_at"))
.limit(limit)
)
rows = (await session.execute(stmt)).all()
return [
AdminLeaderboardQuestion(
text=(r[0] or "")[:500],
user_id=r[2] or "",
email=r[3] or "",
asked_at=_iso(r[4]),
)
for r in rows
]
async def _query_trends(session, since: datetime, until: datetime, *, user_id: str | None = None) -> list[AdminLeaderboardTrendPoint]:
bucket = func.date(RunRow.created_at)
stmt = (
select(
bucket.label("date"),
func.count(RunRow.run_id).label("runs"),
_question_count_expr().label("questions"),
func.coalesce(func.sum(RunRow.total_tokens), 0).label("tokens"),
func.count(func.distinct(RunRow.user_id)).label("users"),
)
.where(_run_time_filter(since, until))
.group_by(bucket)
.order_by(bucket.asc())
)
if user_id:
stmt = stmt.where(RunRow.user_id == user_id)
rows = (await session.execute(stmt)).all()
return [
AdminLeaderboardTrendPoint(
date=str(r[0]),
runs=int(r[1] or 0),
questions=int(r[2] or 0),
tokens=int(r[3] or 0),
users=int(r[4] or 0),
)
for r in rows
]
async def _query_overview(session, since: datetime, until: datetime, *, user_id: str | None = None) -> AdminLeaderboardOverview:
run_where = [_run_time_filter(since, until)]
thread_where = [and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until)]
tool_where = [_tool_time_filter(since, until)]
if user_id:
run_where.append(RunRow.user_id == user_id)
thread_where.append(ThreadMetaRow.user_id == user_id)
tool_where.append(ToolCallMetricRow.user_id == user_id)
total_users = 1 if user_id else (await session.execute(select(func.count(UserRow.id)))).scalar_one()
run_row = (await session.execute(
select(
func.count(RunRow.run_id),
_question_count_expr(),
func.coalesce(func.sum(RunRow.total_tokens), 0),
func.count(func.distinct(RunRow.user_id)),
func.sum(case((RunRow.status == "error", 1), else_=0)),
).where(*run_where)
)).one()
thread_count = (await session.execute(select(func.count(ThreadMetaRow.thread_id)).where(*thread_where))).scalar_one()
tool_row = (await session.execute(
select(
func.count(ToolCallMetricRow.id),
func.sum(case((ToolCallMetricRow.skill_name.is_not(None), 1), else_=0)),
).where(*tool_where)
)).one()
return AdminLeaderboardOverview(
total_users=int(total_users or 0),
active_users=int(run_row[3] or 0),
total_threads=int(thread_count or 0),
total_runs=int(run_row[0] or 0),
total_questions=int(run_row[1] or 0),
total_tokens=int(run_row[2] or 0),
total_tool_calls=int(tool_row[0] or 0),
total_skill_calls=int(tool_row[1] or 0),
error_runs=int(run_row[4] or 0),
)
@router.get("/leaderboard", response_model=AdminLeaderboardResponse)
async def get_user_leaderboard(
request: Request,
days: int = Query(default=30, ge=1, le=365),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminLeaderboardResponse:
"""Return admin analytics for user activity, Q&A, skills, questions, and trends."""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
until = _now()
since = _range_start(days)
async with session_factory() as session:
overview = await _query_overview(session, since, until)
users = await _query_user_leaderboard(session, since, until)
skills = await _query_call_rankings(session, since, until, user_id=None, field="skill", limit=limit)
tools = await _query_call_rankings(session, since, until, user_id=None, field="tool", limit=limit)
recent_questions = await _query_recent_questions(session, since, until, user_id=None, limit=limit)
frequent_questions = await _query_frequent_questions(session, since, until, limit=limit)
trends = await _query_trends(session, since, until)
return AdminLeaderboardResponse(
range_days=days,
since=_iso(since),
until=_iso(until),
overview=overview,
users_by_activity=sorted(users, key=lambda u: (u.run_count + u.tool_call_count, u.total_tokens), reverse=True)[:limit],
users_by_questions=sorted(users, key=lambda u: (u.qa_count, u.run_count), reverse=True)[:limit],
users_by_tokens=sorted(users, key=lambda u: u.total_tokens, reverse=True)[:limit],
skills=skills,
tools=tools,
recent_questions=recent_questions,
frequent_questions=frequent_questions,
trends=trends,
)
@router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse)
async def get_user_analytics(
user_id: str,
request: Request,
days: int = Query(default=30, ge=1, le=365),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminUserAnalyticsResponse:
"""Return one user's admin analytics profile."""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
until = _now()
since = _range_start(days)
async with session_factory() as session:
users = await _query_user_leaderboard(session, since, until, user_id=user_id)
if not users:
raise HTTPException(status_code=404, detail="User not found")
overview = await _query_overview(session, since, until, user_id=user_id)
skills = await _query_call_rankings(session, since, until, user_id=user_id, field="skill", limit=limit)
tools = await _query_call_rankings(session, since, until, user_id=user_id, field="tool", limit=limit)
recent_questions = await _query_recent_questions(session, since, until, user_id=user_id, limit=limit)
trends = await _query_trends(session, since, until, user_id=user_id)
return AdminUserAnalyticsResponse(
range_days=days,
since=_iso(since),
until=_iso(until),
user=users[0],
overview=overview,
skills=skills,
tools=tools,
recent_questions=recent_questions,
trends=trends,
)
# ── Threads ────────────────────────────────────────────────────────────────