feat: add admin user leaderboard analytics
This commit is contained in:
1 parent
1e4ef070cf
commit
8f80b7c1ef
7 files changed
+1060
-1
No files matched your search
@@ -15,8 +15,11 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import and_, case, desc, func, select
|
||||
|
||||
from app.gateway.deps import (
|
||||
get_agent_store,
|
||||
@@ -28,6 +31,11 @@ from deerflow.agents.memory import BuiltinFileProvider
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from app.gateway.routers.memory import _build_v1_provider, _is_v1
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/admin/users", tags=["admin-users"])
|
||||
@@ -45,6 +53,429 @@ async def _require_admin(request: Request) -> None:
|
||||
raise HTTPException(status_code=403, detail="Admin only")
|
||||
|
||||
|
||||
# ── Leaderboard / analytics ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class AdminLeaderboardUser(BaseModel):
|
||||
user_id: str
|
||||
email: str = ""
|
||||
system_role: str = "user"
|
||||
thread_count: int = 0
|
||||
run_count: int = 0
|
||||
qa_count: int = 0
|
||||
llm_call_count: int = 0
|
||||
tool_call_count: int = 0
|
||||
skill_call_count: int = 0
|
||||
total_tokens: int = 0
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
last_active_at: str = ""
|
||||
|
||||
|
||||
class AdminLeaderboardSkill(BaseModel):
|
||||
name: str
|
||||
call_count: int = 0
|
||||
user_count: int = 0
|
||||
success_count: int = 0
|
||||
error_count: int = 0
|
||||
avg_duration_ms: int = 0
|
||||
last_used_at: str = ""
|
||||
|
||||
|
||||
class AdminLeaderboardQuestion(BaseModel):
|
||||
text: str
|
||||
user_id: str = ""
|
||||
email: str = ""
|
||||
thread_id: str = ""
|
||||
run_id: str = ""
|
||||
asked_at: str = ""
|
||||
|
||||
|
||||
class AdminLeaderboardTrendPoint(BaseModel):
|
||||
date: str
|
||||
runs: int = 0
|
||||
questions: int = 0
|
||||
tokens: int = 0
|
||||
users: int = 0
|
||||
|
||||
|
||||
class AdminLeaderboardOverview(BaseModel):
|
||||
total_users: int = 0
|
||||
active_users: int = 0
|
||||
total_threads: int = 0
|
||||
total_runs: int = 0
|
||||
total_questions: int = 0
|
||||
total_tokens: int = 0
|
||||
total_skill_calls: int = 0
|
||||
total_tool_calls: int = 0
|
||||
error_runs: int = 0
|
||||
|
||||
|
||||
class AdminLeaderboardResponse(BaseModel):
|
||||
range_days: int
|
||||
since: str
|
||||
until: str
|
||||
overview: AdminLeaderboardOverview
|
||||
users_by_activity: list[AdminLeaderboardUser] = Field(default_factory=list)
|
||||
users_by_questions: list[AdminLeaderboardUser] = Field(default_factory=list)
|
||||
users_by_tokens: list[AdminLeaderboardUser] = Field(default_factory=list)
|
||||
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
frequent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AdminUserAnalyticsResponse(BaseModel):
|
||||
range_days: int
|
||||
since: str
|
||||
until: str
|
||||
user: AdminLeaderboardUser
|
||||
overview: AdminLeaderboardOverview
|
||||
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list)
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now()
|
||||
|
||||
|
||||
def _iso(value) -> str:
|
||||
return value.isoformat() if isinstance(value, datetime) else str(value or "")
|
||||
|
||||
|
||||
def _range_start(days: int) -> datetime:
|
||||
return _now() - timedelta(days=max(1, min(days, 365)))
|
||||
|
||||
|
||||
def _question_count_expr():
|
||||
return func.sum(case((RunRow.first_human_message.is_not(None), 1), else_=0))
|
||||
|
||||
|
||||
def _run_time_filter(since: datetime, until: datetime):
|
||||
return and_(RunRow.created_at >= since, RunRow.created_at <= until)
|
||||
|
||||
|
||||
def _tool_time_filter(since: datetime, until: datetime):
|
||||
return and_(ToolCallMetricRow.created_at >= since, ToolCallMetricRow.created_at <= until)
|
||||
|
||||
|
||||
async def _query_user_leaderboard(session, since: datetime, until: datetime, user_id: str | None = None) -> list[AdminLeaderboardUser]:
|
||||
run_agg = (
|
||||
select(
|
||||
RunRow.user_id.label("user_id"),
|
||||
func.count(RunRow.run_id).label("run_count"),
|
||||
_question_count_expr().label("qa_count"),
|
||||
func.coalesce(func.sum(RunRow.llm_call_count), 0).label("llm_call_count"),
|
||||
func.coalesce(func.sum(RunRow.total_tokens), 0).label("total_tokens"),
|
||||
func.coalesce(func.sum(RunRow.total_input_tokens), 0).label("input_tokens"),
|
||||
func.coalesce(func.sum(RunRow.total_output_tokens), 0).label("output_tokens"),
|
||||
func.max(RunRow.updated_at).label("last_active_at"),
|
||||
)
|
||||
.where(_run_time_filter(since, until))
|
||||
.group_by(RunRow.user_id)
|
||||
.subquery()
|
||||
)
|
||||
thread_agg = (
|
||||
select(ThreadMetaRow.user_id.label("user_id"), func.count(ThreadMetaRow.thread_id).label("thread_count"))
|
||||
.where(and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until))
|
||||
.group_by(ThreadMetaRow.user_id)
|
||||
.subquery()
|
||||
)
|
||||
tool_agg = (
|
||||
select(
|
||||
ToolCallMetricRow.user_id.label("user_id"),
|
||||
func.count(ToolCallMetricRow.id).label("tool_call_count"),
|
||||
func.sum(case((ToolCallMetricRow.skill_name.is_not(None), 1), else_=0)).label("skill_call_count"),
|
||||
)
|
||||
.where(_tool_time_filter(since, until))
|
||||
.group_by(ToolCallMetricRow.user_id)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
stmt = (
|
||||
select(
|
||||
UserRow.id,
|
||||
UserRow.email,
|
||||
UserRow.system_role,
|
||||
func.coalesce(thread_agg.c.thread_count, 0),
|
||||
func.coalesce(run_agg.c.run_count, 0),
|
||||
func.coalesce(run_agg.c.qa_count, 0),
|
||||
func.coalesce(run_agg.c.llm_call_count, 0),
|
||||
func.coalesce(tool_agg.c.tool_call_count, 0),
|
||||
func.coalesce(tool_agg.c.skill_call_count, 0),
|
||||
func.coalesce(run_agg.c.total_tokens, 0),
|
||||
func.coalesce(run_agg.c.input_tokens, 0),
|
||||
func.coalesce(run_agg.c.output_tokens, 0),
|
||||
run_agg.c.last_active_at,
|
||||
)
|
||||
.outerjoin(run_agg, run_agg.c.user_id == UserRow.id)
|
||||
.outerjoin(thread_agg, thread_agg.c.user_id == UserRow.id)
|
||||
.outerjoin(tool_agg, tool_agg.c.user_id == UserRow.id)
|
||||
)
|
||||
if user_id:
|
||||
stmt = stmt.where(UserRow.id == user_id)
|
||||
stmt = stmt.order_by(
|
||||
desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)),
|
||||
desc(func.coalesce(run_agg.c.total_tokens, 0)),
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardUser(
|
||||
user_id=r[0],
|
||||
email=r[1] or "",
|
||||
system_role=r[2] or "user",
|
||||
thread_count=int(r[3] or 0),
|
||||
run_count=int(r[4] or 0),
|
||||
qa_count=int(r[5] or 0),
|
||||
llm_call_count=int(r[6] or 0),
|
||||
tool_call_count=int(r[7] or 0),
|
||||
skill_call_count=int(r[8] or 0),
|
||||
total_tokens=int(r[9] or 0),
|
||||
input_tokens=int(r[10] or 0),
|
||||
output_tokens=int(r[11] or 0),
|
||||
last_active_at=_iso(r[12]),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _query_call_rankings(session, since: datetime, until: datetime, *, user_id: str | None, field: str, limit: int) -> list[AdminLeaderboardSkill]:
|
||||
name_col = ToolCallMetricRow.skill_name if field == "skill" else ToolCallMetricRow.tool_name
|
||||
call_count = func.count(ToolCallMetricRow.id).label("call_count")
|
||||
last_used_at = func.max(ToolCallMetricRow.created_at).label("last_used_at")
|
||||
stmt = (
|
||||
select(
|
||||
name_col.label("name"),
|
||||
call_count,
|
||||
func.count(func.distinct(ToolCallMetricRow.user_id)).label("user_count"),
|
||||
func.sum(case((ToolCallMetricRow.status == "success", 1), else_=0)).label("success_count"),
|
||||
func.sum(case((ToolCallMetricRow.status != "success", 1), else_=0)).label("error_count"),
|
||||
func.coalesce(func.avg(ToolCallMetricRow.duration_ms), 0).label("avg_duration_ms"),
|
||||
last_used_at,
|
||||
)
|
||||
.where(_tool_time_filter(since, until), name_col.is_not(None), name_col != "")
|
||||
.group_by(name_col)
|
||||
.order_by(call_count.desc(), last_used_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if user_id:
|
||||
stmt = stmt.where(ToolCallMetricRow.user_id == user_id)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardSkill(
|
||||
name=r[0] or "",
|
||||
call_count=int(r[1] or 0),
|
||||
user_count=int(r[2] or 0),
|
||||
success_count=int(r[3] or 0),
|
||||
error_count=int(r[4] or 0),
|
||||
avg_duration_ms=int(r[5] or 0),
|
||||
last_used_at=_iso(r[6]),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _query_recent_questions(session, since: datetime, until: datetime, *, user_id: str | None, limit: int) -> list[AdminLeaderboardQuestion]:
|
||||
stmt = (
|
||||
select(RunRow.first_human_message, RunRow.user_id, UserRow.email, RunRow.thread_id, RunRow.run_id, RunRow.created_at)
|
||||
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
|
||||
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
|
||||
.order_by(RunRow.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if user_id:
|
||||
stmt = stmt.where(RunRow.user_id == user_id)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardQuestion(
|
||||
text=(r[0] or "")[:500],
|
||||
user_id=r[1] or "",
|
||||
email=r[2] or "",
|
||||
thread_id=r[3] or "",
|
||||
run_id=r[4] or "",
|
||||
asked_at=_iso(r[5]),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _query_frequent_questions(session, since: datetime, until: datetime, limit: int) -> list[AdminLeaderboardQuestion]:
|
||||
stmt = (
|
||||
select(
|
||||
RunRow.first_human_message,
|
||||
func.count(RunRow.run_id).label("count"),
|
||||
func.max(RunRow.user_id).label("user_id"),
|
||||
func.max(UserRow.email).label("email"),
|
||||
func.max(RunRow.created_at).label("asked_at"),
|
||||
)
|
||||
.outerjoin(UserRow, UserRow.id == RunRow.user_id)
|
||||
.where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "")
|
||||
.group_by(RunRow.first_human_message)
|
||||
.having(func.count(RunRow.run_id) > 1)
|
||||
.order_by(desc("count"), desc("asked_at"))
|
||||
.limit(limit)
|
||||
)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardQuestion(
|
||||
text=(r[0] or "")[:500],
|
||||
user_id=r[2] or "",
|
||||
email=r[3] or "",
|
||||
asked_at=_iso(r[4]),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _query_trends(session, since: datetime, until: datetime, *, user_id: str | None = None) -> list[AdminLeaderboardTrendPoint]:
|
||||
bucket = func.date(RunRow.created_at)
|
||||
stmt = (
|
||||
select(
|
||||
bucket.label("date"),
|
||||
func.count(RunRow.run_id).label("runs"),
|
||||
_question_count_expr().label("questions"),
|
||||
func.coalesce(func.sum(RunRow.total_tokens), 0).label("tokens"),
|
||||
func.count(func.distinct(RunRow.user_id)).label("users"),
|
||||
)
|
||||
.where(_run_time_filter(since, until))
|
||||
.group_by(bucket)
|
||||
.order_by(bucket.asc())
|
||||
)
|
||||
if user_id:
|
||||
stmt = stmt.where(RunRow.user_id == user_id)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return [
|
||||
AdminLeaderboardTrendPoint(
|
||||
date=str(r[0]),
|
||||
runs=int(r[1] or 0),
|
||||
questions=int(r[2] or 0),
|
||||
tokens=int(r[3] or 0),
|
||||
users=int(r[4] or 0),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _query_overview(session, since: datetime, until: datetime, *, user_id: str | None = None) -> AdminLeaderboardOverview:
|
||||
run_where = [_run_time_filter(since, until)]
|
||||
thread_where = [and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until)]
|
||||
tool_where = [_tool_time_filter(since, until)]
|
||||
if user_id:
|
||||
run_where.append(RunRow.user_id == user_id)
|
||||
thread_where.append(ThreadMetaRow.user_id == user_id)
|
||||
tool_where.append(ToolCallMetricRow.user_id == user_id)
|
||||
|
||||
total_users = 1 if user_id else (await session.execute(select(func.count(UserRow.id)))).scalar_one()
|
||||
run_row = (await session.execute(
|
||||
select(
|
||||
func.count(RunRow.run_id),
|
||||
_question_count_expr(),
|
||||
func.coalesce(func.sum(RunRow.total_tokens), 0),
|
||||
func.count(func.distinct(RunRow.user_id)),
|
||||
func.sum(case((RunRow.status == "error", 1), else_=0)),
|
||||
).where(*run_where)
|
||||
)).one()
|
||||
thread_count = (await session.execute(select(func.count(ThreadMetaRow.thread_id)).where(*thread_where))).scalar_one()
|
||||
tool_row = (await session.execute(
|
||||
select(
|
||||
func.count(ToolCallMetricRow.id),
|
||||
func.sum(case((ToolCallMetricRow.skill_name.is_not(None), 1), else_=0)),
|
||||
).where(*tool_where)
|
||||
)).one()
|
||||
|
||||
return AdminLeaderboardOverview(
|
||||
total_users=int(total_users or 0),
|
||||
active_users=int(run_row[3] or 0),
|
||||
total_threads=int(thread_count or 0),
|
||||
total_runs=int(run_row[0] or 0),
|
||||
total_questions=int(run_row[1] or 0),
|
||||
total_tokens=int(run_row[2] or 0),
|
||||
total_tool_calls=int(tool_row[0] or 0),
|
||||
total_skill_calls=int(tool_row[1] or 0),
|
||||
error_runs=int(run_row[4] or 0),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/leaderboard", response_model=AdminLeaderboardResponse)
|
||||
async def get_user_leaderboard(
|
||||
request: Request,
|
||||
days: int = Query(default=30, ge=1, le=365),
|
||||
limit: int = Query(default=20, ge=5, le=100),
|
||||
) -> AdminLeaderboardResponse:
|
||||
"""Return admin analytics for user activity, Q&A, skills, questions, and trends."""
|
||||
await _require_admin(request)
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
|
||||
|
||||
until = _now()
|
||||
since = _range_start(days)
|
||||
async with session_factory() as session:
|
||||
overview = await _query_overview(session, since, until)
|
||||
users = await _query_user_leaderboard(session, since, until)
|
||||
skills = await _query_call_rankings(session, since, until, user_id=None, field="skill", limit=limit)
|
||||
tools = await _query_call_rankings(session, since, until, user_id=None, field="tool", limit=limit)
|
||||
recent_questions = await _query_recent_questions(session, since, until, user_id=None, limit=limit)
|
||||
frequent_questions = await _query_frequent_questions(session, since, until, limit=limit)
|
||||
trends = await _query_trends(session, since, until)
|
||||
|
||||
return AdminLeaderboardResponse(
|
||||
range_days=days,
|
||||
since=_iso(since),
|
||||
until=_iso(until),
|
||||
overview=overview,
|
||||
users_by_activity=sorted(users, key=lambda u: (u.run_count + u.tool_call_count, u.total_tokens), reverse=True)[:limit],
|
||||
users_by_questions=sorted(users, key=lambda u: (u.qa_count, u.run_count), reverse=True)[:limit],
|
||||
users_by_tokens=sorted(users, key=lambda u: u.total_tokens, reverse=True)[:limit],
|
||||
skills=skills,
|
||||
tools=tools,
|
||||
recent_questions=recent_questions,
|
||||
frequent_questions=frequent_questions,
|
||||
trends=trends,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse)
|
||||
async def get_user_analytics(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
days: int = Query(default=30, ge=1, le=365),
|
||||
limit: int = Query(default=20, ge=5, le=100),
|
||||
) -> AdminUserAnalyticsResponse:
|
||||
"""Return one user's admin analytics profile."""
|
||||
await _require_admin(request)
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
|
||||
|
||||
until = _now()
|
||||
since = _range_start(days)
|
||||
async with session_factory() as session:
|
||||
users = await _query_user_leaderboard(session, since, until, user_id=user_id)
|
||||
if not users:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
overview = await _query_overview(session, since, until, user_id=user_id)
|
||||
skills = await _query_call_rankings(session, since, until, user_id=user_id, field="skill", limit=limit)
|
||||
tools = await _query_call_rankings(session, since, until, user_id=user_id, field="tool", limit=limit)
|
||||
recent_questions = await _query_recent_questions(session, since, until, user_id=user_id, limit=limit)
|
||||
trends = await _query_trends(session, since, until, user_id=user_id)
|
||||
|
||||
return AdminUserAnalyticsResponse(
|
||||
range_days=days,
|
||||
since=_iso(since),
|
||||
until=_iso(until),
|
||||
user=users[0],
|
||||
overview=overview,
|
||||
skills=skills,
|
||||
tools=tools,
|
||||
recent_questions=recent_questions,
|
||||
trends=trends,
|
||||
)
|
||||
|
||||
|
||||
# ── Threads ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user