From fe0ede178e6924400b2f62e80bbce2a8a9c6df9b Mon Sep 17 00:00:00 2001 From: Administrator Date: Thu, 4 Jun 2026 21:09:36 +0800 Subject: [PATCH] polish admin leaderboard analytics --- frontend-web/src/core/admin/api.ts | 32 +- frontend-web/src/core/admin/hooks.ts | 22 +- .../src/pages/AdminUserLeaderboardPage.tsx | 751 ++++++++++++++++-- .../app/gateway/routers/admin_users.py | 403 +++++++++- 4 files changed, 1122 insertions(+), 86 deletions(-) diff --git a/frontend-web/src/core/admin/api.ts b/frontend-web/src/core/admin/api.ts index a8fb2df9..f27cb0d1 100644 --- a/frontend-web/src/core/admin/api.ts +++ b/frontend-web/src/core/admin/api.ts @@ -61,6 +61,16 @@ export interface AdminLeaderboardSkill { last_used_at: string; } +export interface AdminLeaderboardDimension { + name: string; + count: number; + user_count: number; + success_count: number; + error_count: number; + total_tokens: number; + last_seen_at: string; +} + export interface AdminLeaderboardQuestion { text: string; user_id: string; @@ -100,6 +110,9 @@ export interface AdminLeaderboardResponse { users_by_tokens: AdminLeaderboardUser[]; skills: AdminLeaderboardSkill[]; tools: AdminLeaderboardSkill[]; + models: AdminLeaderboardDimension[]; + agents: AdminLeaderboardDimension[]; + run_statuses: AdminLeaderboardDimension[]; recent_questions: AdminLeaderboardQuestion[]; frequent_questions: AdminLeaderboardQuestion[]; trends: AdminLeaderboardTrendPoint[]; @@ -113,6 +126,9 @@ export interface AdminUserAnalyticsResponse { overview: AdminLeaderboardOverview; skills: AdminLeaderboardSkill[]; tools: AdminLeaderboardSkill[]; + models: AdminLeaderboardDimension[]; + agents: AdminLeaderboardDimension[]; + run_statuses: AdminLeaderboardDimension[]; recent_questions: AdminLeaderboardQuestion[]; trends: AdminLeaderboardTrendPoint[]; } @@ -156,7 +172,15 @@ export async function getUserMemory( } export async function getUserLeaderboard( - range: { since: string; until: string; days?: number }, + range: { + since: string; + until: string; + days?: number; + exclude_user_ids?: string[]; + include_scheduled?: boolean; + include_admins?: boolean; + include_failed?: boolean; + }, limit = 20, ): Promise { const params = new URLSearchParams({ @@ -164,7 +188,13 @@ export async function getUserLeaderboard( since: range.since, until: range.until, limit: String(limit), + include_scheduled: String(range.include_scheduled ?? true), + include_admins: String(range.include_admins ?? true), + include_failed: String(range.include_failed ?? true), }); + for (const userId of range.exclude_user_ids ?? []) { + params.append("exclude_user_ids", userId); + } return parse( await apiFetch(`${base()}/leaderboard?${params}`), "Failed to load leaderboard", diff --git a/frontend-web/src/core/admin/hooks.ts b/frontend-web/src/core/admin/hooks.ts index b041f33f..c6ffdc2e 100644 --- a/frontend-web/src/core/admin/hooks.ts +++ b/frontend-web/src/core/admin/hooks.ts @@ -16,11 +16,29 @@ import { } from "./api"; export function useUserLeaderboard( - range: { since: string; until: string; days?: number }, + range: { + since: string; + until: string; + days?: number; + exclude_user_ids?: string[]; + include_scheduled?: boolean; + include_admins?: boolean; + include_failed?: boolean; + }, limit = 20, ) { return useQuery({ - queryKey: ["admin-activity", "leaderboard", range.since, range.until, limit], + queryKey: [ + "admin-activity", + "leaderboard", + range.since, + range.until, + range.exclude_user_ids ?? [], + range.include_scheduled ?? true, + range.include_admins ?? true, + range.include_failed ?? true, + limit, + ], queryFn: () => getUserLeaderboard(range, limit), }); } diff --git a/frontend-web/src/pages/AdminUserLeaderboardPage.tsx b/frontend-web/src/pages/AdminUserLeaderboardPage.tsx index e0f37d09..8c27f297 100644 --- a/frontend-web/src/pages/AdminUserLeaderboardPage.tsx +++ b/frontend-web/src/pages/AdminUserLeaderboardPage.tsx @@ -2,72 +2,97 @@ import { Activity, + ArrowLeft, ArrowRight, Award, BarChart3, + Bot, CalendarDays, Crown, + Gauge, + Layers3, Loader2, + Maximize2, Medal, MessageSquareText, MousePointerClick, Search, + ShieldOff, + Settings2, Sparkles, TrendingUp, UserRound, Users, + X, type LucideIcon, } from "lucide-react"; -import { useMemo, useState } from "react"; +import { useEffect, useId, useMemo, useState } from "react"; import { useNavigate } from "react-router-dom"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { ScrollArea } from "@/components/ui/scroll-area"; +import { Switch } from "@/components/ui/switch"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { + AdminLeaderboardDimension, AdminLeaderboardQuestion, AdminLeaderboardSkill, AdminLeaderboardTrendPoint, AdminLeaderboardUser, } from "@/core/admin/api"; import { useUserAnalytics, useUserLeaderboard } from "@/core/admin/hooks"; +import type { UserListItem } from "@/core/auth/api"; +import { useAllUsers } from "@/core/auth/hooks"; import { cn } from "@/lib/utils"; -type UserRankTab = "activity" | "questions" | "tokens"; +type UserRankTab = "activity" | "questions" | "tokens" | "skills" | "efficiency"; type RangeState = { since: string; until: string; days?: number }; -type GoogleAccent = "blue" | "red" | "yellow" | "green"; +type StatsConfig = { + include_scheduled: boolean; + include_admins: boolean; + include_failed: boolean; +}; +type LeaderboardAccent = "indigo" | "rose" | "amber" | "teal"; -const GOOGLE_COLORS = ["#4285F4", "#DB4437", "#F4B400", "#0F9D58"] as const; - -const accentClasses: Record = { - blue: { - soft: "bg-[#E8F0FE] dark:bg-[#4285F4]/15", - text: "text-[#4285F4] dark:text-[#8AB4F8]", - bar: "bg-[#4285F4]", - border: "border-[#D2E3FC] dark:border-[#4285F4]/35", +const accentClasses: Record = { + indigo: { + soft: "bg-[#EEF2FF] dark:bg-indigo-400/15", + text: "text-[#4F46E5] dark:text-indigo-200", + bar: "bg-[#4F46E5]", + border: "border-[#C7D2FE] dark:border-indigo-300/25", }, - red: { - soft: "bg-[#FCE8E6] dark:bg-[#DB4437]/15", - text: "text-[#DB4437] dark:text-[#F28B82]", - bar: "bg-[#DB4437]", - border: "border-[#FAD2CF] dark:border-[#DB4437]/35", + rose: { + soft: "bg-[#FFF1F2] dark:bg-rose-400/15", + text: "text-[#E11D48] dark:text-rose-200", + bar: "bg-[#E11D48]", + border: "border-[#FECDD3] dark:border-rose-300/25", }, - yellow: { - soft: "bg-[#FEF7E0] dark:bg-[#F4B400]/15", - text: "text-[#B06000] dark:text-[#FDD663]", - bar: "bg-[#F4B400]", - border: "border-[#FEEFC3] dark:border-[#F4B400]/35", + amber: { + soft: "bg-[#FFFBEB] dark:bg-amber-300/15", + text: "text-[#B45309] dark:text-amber-100", + bar: "bg-[#F59E0B]", + border: "border-[#FDE68A] dark:border-amber-200/25", }, - green: { - soft: "bg-[#E6F4EA] dark:bg-[#0F9D58]/15", - text: "text-[#0F9D58] dark:text-[#81C995]", - bar: "bg-[#0F9D58]", - border: "border-[#CEEAD6] dark:border-[#0F9D58]/35", + teal: { + soft: "bg-[#ECFDF5] dark:bg-teal-300/15", + text: "text-[#0F766E] dark:text-teal-100", + bar: "bg-[#0F766E]", + border: "border-[#A7F3D0] dark:border-teal-200/25", }, }; +const RANK_COLORS = ["#4F46E5", "#0F766E", "#F59E0B", "#E11D48", "#7C3AED"] as const; + function toDateInput(date: Date): string { const year = date.getFullYear(); const month = String(date.getMonth() + 1).padStart(2, "0"); @@ -111,15 +136,31 @@ function clampText(value: string, max = 120): string { function userScore(user: AdminLeaderboardUser, tab: UserRankTab): number { if (tab === "questions") return user.qa_count; if (tab === "tokens") return user.total_tokens; + if (tab === "skills") return user.skill_call_count; + if (tab === "efficiency") { + return user.run_count > 0 ? Math.round((user.qa_count / user.run_count) * 100) : 0; + } return user.run_count + user.tool_call_count; } function scoreLabel(user: AdminLeaderboardUser, tab: UserRankTab): string { if (tab === "questions") return `${fmtNumber(user.qa_count)} 次问答`; if (tab === "tokens") return `${fmtNumber(user.total_tokens)} tokens`; + if (tab === "skills") return `${fmtNumber(user.skill_call_count)} 次技能`; + if (tab === "efficiency") { + return `${(user.run_count > 0 ? user.qa_count / user.run_count : 0).toFixed(1)} 问答/运行`; + } return `${fmtNumber(userScore(user, tab))} 次使用`; } +function rankTabDescription(tab: UserRankTab): string { + if (tab === "questions") return "问答排行按用户提出有效问题的运行次数排序。"; + if (tab === "tokens") return "消耗排行按模型输入与输出 Token 总量排序。"; + if (tab === "skills") return "技能排行按带有技能名称的工具调用次数排序。"; + if (tab === "efficiency") return "效率排行 = 问答次数 / 运行次数，用来衡量每次运行承载的问答密度。"; + return "使用排行按运行次数与工具调用次数合计排序。"; +} + function initials(user: AdminLeaderboardUser): string { const source = user.email || user.user_id || "?"; return source.slice(0, 1).toUpperCase(); @@ -136,7 +177,7 @@ function StatCard({ label: string; value: string; note: string; - accent: GoogleAccent; + accent: LeaderboardAccent; }) { const color = accentClasses[accent]; return ( @@ -215,25 +256,295 @@ function PodiumCard({ ); } -function TrendBars({ points }: { points: AdminLeaderboardTrendPoint[] }) { - const max = Math.max(1, ...points.map((p) => p.runs)); - if (points.length === 0) { - return
暂无趋势数据
; +function SpotlightCarousel({ + users, + tab, + onSelect, +}: { + users: AdminLeaderboardUser[]; + tab: UserRankTab; + onSelect: (user: AdminLeaderboardUser) => void; +}) { + const [index, setIndex] = useState(0); + const visible = users.slice(0, 5); + const active = visible[index % Math.max(visible.length, 1)]; + + useEffect(() => { + if (visible.length <= 1) return; + const timer = window.setInterval(() => { + setIndex((value) => (value + 1) % visible.length); + }, 5000); + return () => window.clearInterval(timer); + }, [visible.length]); + + useEffect(() => { + if (index >= visible.length) setIndex(0); + }, [index, visible.length]); + + if (!active) { + return ( +
+
暂无可轮播用户
+
+ ); } + + const maxScore = Math.max(1, ...visible.map((user) => userScore(user, tab))); + return ( -
- {points.slice(-32).map((point, index) => ( -
-
+
+
+ +
+
+
焦点队列
+
自动轮播 Top 5
+
+
+ {visible.map((user, itemIndex) => ( + + ))} +
+
+ {visible.map((user, itemIndex) => ( +
- ))} +
+
+ ); +} + +function TrendBars({ points }: { points: AdminLeaderboardTrendPoint[] }) { + const [metric, setMetric] = useState>("runs"); + const chartId = useId().replace(/:/g, ""); + if (points.length === 0) { + return
暂无趋势数据
; + } + + const visible = points.slice(-32); + const series = [ + { key: "runs", label: "运行", color: "#4F46E5", soft: "bg-[#EEF2FF] text-[#4F46E5]" }, + { key: "questions", label: "问答", color: "#0F766E", soft: "bg-[#ECFDF5] text-[#0F766E]" }, + { key: "users", label: "活跃用户", color: "#B45309", soft: "bg-[#FFFBEB] text-[#B45309]" }, + { key: "tokens", label: "Token", color: "#E11D48", soft: "bg-[#FFF1F2] text-[#E11D48]" }, + ] as const; + const activeSeries = series.find((item) => item.key === metric) ?? series[0]; + const metricMax = Math.max(1, ...visible.map((point) => Number(point[metric] ?? 0))); + const latest = visible[visible.length - 1]; + const previous = visible[visible.length - 2]; + const latestValue = Number(latest?.[metric] ?? 0); + const previousValue = Number(previous?.[metric] ?? 0); + const delta = latestValue - previousValue; + const width = 920; + const height = 280; + const padding = { top: 22, right: 28, bottom: 36, left: 48 }; + const chartWidth = width - padding.left - padding.right; + const chartHeight = height - padding.top - padding.bottom; + const xFor = (index: number) => + padding.left + (visible.length <= 1 ? chartWidth / 2 : (index / (visible.length - 1)) * chartWidth); + const yFor = (value: number, max: number) => padding.top + chartHeight - (value / max) * chartHeight; + const gridLines = [0, 0.25, 0.5, 0.75, 1]; + const selectedPath = visible + .map((point, index) => { + const value = Number(point[metric] ?? 0); + return `${index === 0 ? "M" : "L"} ${xFor(index).toFixed(2)} ${yFor(value, metricMax).toFixed(2)}`; + }) + .join(" "); + const selectedArea = `${selectedPath} L ${xFor(visible.length - 1).toFixed(2)} ${padding.top + chartHeight} L ${xFor(0).toFixed(2)} ${padding.top + chartHeight} Z`; + const peak = visible.reduce( + (best, point) => (Number(point[metric] ?? 0) > Number(best[metric] ?? 0) ? point : best), + visible[0], + ); + const chart = (large = false) => ( +
+ + + + + + + + {gridLines.map((ratio) => { + const y = padding.top + chartHeight * ratio; + return ( + + + + {fmtNumber(Math.round(metricMax * (1 - ratio)))} + + + ); + })} + + + {visible.map((point, index) => { + const value = Number(point[metric] ?? 0); + const isPeak = point.date === peak.date; + const isLast = index === visible.length - 1; + return ( + + {`${point.date} ${activeSeries.label}: ${fmtNumber(value)}`} + + ); + })} + {visible.map((point, index) => { + if (index !== 0 && index !== visible.length - 1 && index % Math.ceil(visible.length / 6) !== 0) return null; + return ( + + {point.date.slice(5)} + + ); + })} + +
+ ); + + return ( +
+
+
+ {series.map((item) => ( + + ))} +
+ + + + + + + 时间趋势分析 + + 当前查看 {activeSeries.label}，峰值 {peak.date} / {fmtNumber(Number(peak[metric] ?? 0))}。 + + + {chart(true)} + + +
+
+
+
最新 {activeSeries.label}
+
{fmtNumber(latestValue)}
+
= 0 ? "text-[#0F766E]" : "text-[#E11D48]")}> + 较上一日 {delta >= 0 ? "+" : ""}{fmtNumber(delta)} +
+
+ 峰值 {peak.date.slice(5)} / {fmtNumber(Number(peak[metric] ?? 0))} +
+
+ {chart(false)} +
); } @@ -342,7 +653,7 @@ function SkillRanking({ className="h-2 rounded-full" style={{ width: `${Math.max(4, (item.call_count / max) * 100)}%`, - backgroundColor: GOOGLE_COLORS[index % GOOGLE_COLORS.length], + backgroundColor: RANK_COLORS[index % RANK_COLORS.length], }} />
@@ -362,6 +673,63 @@ function SkillRanking({ ); } +function DimensionRanking({ + title, + icon: Icon, + items, + empty, +}: { + title: string; + icon: LucideIcon; + items: AdminLeaderboardDimension[]; + empty: string; +}) { + const max = Math.max(1, ...items.map((item) => item.count)); + return ( +
+
+
+ +

{title}

+
+ {items.length} 项 +
+
+ {items.length === 0 &&
{empty}
} + {items.slice(0, 8).map((item, index) => ( +
+ {index + 1} +
+
+ {item.name} + + {fmtNumber(item.user_count)} 人 + +
+
+
+
+
+ 成功 {fmtNumber(item.success_count)} · 失败 {fmtNumber(item.error_count)} · {fmtNumber(item.total_tokens)} tokens +
+
+
+
{fmtNumber(item.count)}
+
次
+
+
+ ))} +
+
+ ); +} + function QuestionPanel({ title, items, @@ -372,7 +740,7 @@ function QuestionPanel({ showUser?: boolean; }) { return ( -
+
@@ -380,19 +748,21 @@ function QuestionPanel({
{items.length}
-
- {items.length === 0 &&
暂无问题记录
} - {items.slice(0, 8).map((item, index) => ( -
-
{clampText(item.text, 180)}
-
- {showUser && {item.email || item.user_id || "未知用户"}} - {fmtTime(item.asked_at)} - {item.thread_id && 会话 {item.thread_id.slice(0, 8)}} + +
+ {items.length === 0 &&
暂无问题记录
} + {items.map((item, index) => ( +
+
{clampText(item.text, 180)}
+
+ {showUser && {item.email || item.user_id || "未知用户"}} + {fmtTime(item.asked_at)} + {item.thread_id && 会话 {item.thread_id.slice(0, 8)}} +
-
- ))} -
+ ))} +
+
); } @@ -476,22 +846,211 @@ function UserDetailPanel({ ); } +function ExclusionSelector({ + users, + excludedUserIds, + query, + onQueryChange, + onAdd, + onRemove, + onClear, +}: { + users: UserListItem[]; + excludedUserIds: string[]; + query: string; + onQueryChange: (value: string) => void; + onAdd: (userId: string) => void; + onRemove: (userId: string) => void; + onClear: () => void; +}) { + const excluded = new Set(excludedUserIds); + const selectedUsers = users.filter((user) => excluded.has(user.id)); + const candidates = users + .filter((user) => !excluded.has(user.id)) + .filter((user) => { + const needle = query.trim().toLowerCase(); + if (!needle) return false; + return `${user.email} ${user.id}`.toLowerCase().includes(needle); + }) + .slice(0, 6); + + return ( +
+
+
+
+ + 统计排除名单 +
+

+ 名单中的账号不参与当前排行榜、KPI、趋势和问题统计。 +

+
+ {excludedUserIds.length > 0 && ( + + )} +
+ +
+
+ + onQueryChange(event.target.value)} + placeholder="搜索账号邮箱或 ID" + className="h-9 bg-white pl-8 dark:bg-[#0B2A50]" + /> + {query.trim() && candidates.length > 0 && ( +
+ + {candidates.map((user) => ( + + ))} + +
+ )} +
+ +
+ {selectedUsers.length === 0 ? ( + 暂无排除账号 + ) : ( + selectedUsers.map((user) => ( + + {user.email} + + + )) + )} +
+
+
+ ); +} + +function StatsRulePanel({ + config, + onChange, + excludedCount, +}: { + config: StatsConfig; + onChange: (patch: Partial) => void; + excludedCount: number; +}) { + const rules: Array<{ + key: keyof StatsConfig; + title: string; + description: string; + }> = [ + { + key: "include_scheduled", + title: "统计定时任务问答", + description: "开启后，定时任务触发的运行、问答、Token、模型调用会进入榜单。", + }, + { + key: "include_admins", + title: "统计管理员账号", + description: "关闭后，管理员账号不会参与排行榜和整体指标。", + }, + { + key: "include_failed", + title: "统计失败运行", + description: "关闭后，失败运行不进入使用、问答、趋势和维度排行。", + }, + ]; + + return ( +
+
+
+
+ + 统计规则 +
+

+ 当前排除 {fmtNumber(excludedCount)} 个账号，以下开关会影响榜单、趋势、模型、技能和问题统计。 +

+
+ 管理员配置 +
+
+ {rules.map((rule) => ( + + ))} +
+
+ ); +} + export default function AdminUserLeaderboardPage() { const navigate = useNavigate(); const [range, setRange] = useState(() => defaultRange(30)); const [tab, setTab] = useState("activity"); const [query, setQuery] = useState(""); + const [excludeQuery, setExcludeQuery] = useState(""); + const [excludedUserIds, setExcludedUserIds] = useState([]); + const [statsConfig, setStatsConfig] = useState({ + include_scheduled: true, + include_admins: true, + include_failed: true, + }); const [selectedUserId, setSelectedUserId] = useState(null); - const apiRange = useMemo(() => rangeForApi(range), [range]); + const { data: allUsers = [] } = useAllUsers(true); + const apiRange = useMemo( + () => ({ ...rangeForApi(range), exclude_user_ids: excludedUserIds, ...statsConfig }), + [range, excludedUserIds, statsConfig], + ); const { data, isLoading, error } = useUserLeaderboard(apiRange, 30); const rankedUsers = useMemo(() => { + const base = data?.users_by_activity ?? []; const source = tab === "questions" ? data?.users_by_questions : tab === "tokens" ? data?.users_by_tokens - : data?.users_by_activity; + : tab === "skills" + ? [...base].sort((a, b) => b.skill_call_count - a.skill_call_count) + : tab === "efficiency" + ? [...base].sort((a, b) => userScore(b, "efficiency") - userScore(a, "efficiency")) + : data?.users_by_activity; const needle = query.trim().toLowerCase(); return (source ?? []).filter((user) => needle ? `${user.email} ${user.user_id}`.toLowerCase().includes(needle) : true, @@ -501,10 +1060,30 @@ export default function AdminUserLeaderboardPage() { const overview = data?.overview; const topUsers = rankedUsers.slice(0, 3); + useEffect(() => { + if (rankedUsers.length === 0) { + setSelectedUserId(null); + return; + } + if (!selectedUserId || !rankedUsers.some((user) => user.user_id === selectedUserId)) { + setSelectedUserId(rankedUsers[0].user_id); + } + }, [rankedUsers, selectedUserId]); + const setQuickRange = (days: number) => { setRange(defaultRange(days)); }; + const addExcludedUser = (userId: string) => { + setExcludedUserIds((current) => + current.includes(userId) ? current : [...current, userId], + ); + }; + + const removeExcludedUser = (userId: string) => { + setExcludedUserIds((current) => current.filter((id) => id !== userId)); + }; + return (
@@ -522,6 +1101,15 @@ export default function AdminUserLeaderboardPage() {
+
+ setExcludedUserIds([])} + /> + setStatsConfig((current) => ({ ...current, ...patch }))} + /> + {isLoading && (
@@ -571,23 +1174,31 @@ export default function AdminUserLeaderboardPage() { {!isLoading && data && overview && ( <>
- - - - + + + +
+ setSelectedUserId(user.user_id)} + /> +

榜单前三

-

根据当前榜单维度实时切换。

+

{rankTabDescription(tab)}

setTab(value as UserRankTab)}> 使用排行 问答排行 消耗排行 + 技能排行 + 效率排行
@@ -603,7 +1214,10 @@ export default function AdminUserLeaderboardPage() {
-

完整排行榜

+
+

完整排行榜

+

{rankTabDescription(tab)}

+
@@ -659,6 +1273,13 @@ export default function AdminUserLeaderboardPage() {
+
+ + + +
+ +
diff --git a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py index 723fe09d..5e2d824d 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py +++ b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py @@ -19,7 +19,7 @@ from datetime import datetime, timedelta from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field -from sqlalchemy import and_, case, desc, func, select +from sqlalchemy import and_, case, desc, func, or_, select from app.gateway.deps import ( get_agent_store, @@ -32,7 +32,9 @@ from deerflow.config.paths import get_paths from deerflow.runtime import serialize_channel_values from app.gateway.routers.memory import _build_v1_provider, _is_v1 from deerflow.persistence.engine import get_session_factory +from deerflow.persistence.llm_metrics.model import LlmCallMetricRow from deerflow.persistence.run.model import RunRow +from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRunRow from deerflow.persistence.thread_meta.model import ThreadMetaRow from deerflow.persistence.tool_metrics.model import ToolCallMetricRow from deerflow.persistence.user.model import UserRow @@ -82,6 +84,16 @@ class AdminLeaderboardSkill(BaseModel): last_used_at: str = "" +class AdminLeaderboardDimension(BaseModel): + name: str + count: int = 0 + user_count: int = 0 + success_count: int = 0 + error_count: int = 0 + total_tokens: int = 0 + last_seen_at: str = "" + + class AdminLeaderboardQuestion(BaseModel): text: str user_id: str = "" @@ -121,6 +133,9 @@ class AdminLeaderboardResponse(BaseModel): users_by_tokens: list[AdminLeaderboardUser] = Field(default_factory=list) skills: list[AdminLeaderboardSkill] = Field(default_factory=list) tools: list[AdminLeaderboardSkill] = Field(default_factory=list) + models: list[AdminLeaderboardDimension] = Field(default_factory=list) + agents: list[AdminLeaderboardDimension] = Field(default_factory=list) + run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list) recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list) frequent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list) trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list) @@ -134,6 +149,9 @@ class AdminUserAnalyticsResponse(BaseModel): overview: AdminLeaderboardOverview skills: list[AdminLeaderboardSkill] = Field(default_factory=list) tools: list[AdminLeaderboardSkill] = Field(default_factory=list) + models: list[AdminLeaderboardDimension] = Field(default_factory=list) + agents: list[AdminLeaderboardDimension] = Field(default_factory=list) + run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list) recent_questions: list[AdminLeaderboardQuestion] = Field(default_factory=list) trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list) @@ -181,7 +199,78 @@ def _tool_time_filter(since: datetime, until: datetime): return and_(ToolCallMetricRow.created_at >= since, ToolCallMetricRow.created_at <= until) -async def _query_user_leaderboard(session, since: datetime, until: datetime, user_id: str | None = None) -> list[AdminLeaderboardUser]: +def _llm_time_filter(since: datetime, until: datetime): + return and_(LlmCallMetricRow.created_at >= since, LlmCallMetricRow.created_at <= until) + + +def _exclude_user_filter(column, exclude_user_ids: set[str]): + if not exclude_user_ids: + return None + return or_(column.is_(None), column.not_in(list(exclude_user_ids))) + + +def _apply_exclude(stmt, column, exclude_user_ids: set[str]): + condition = _exclude_user_filter(column, exclude_user_ids) + return stmt.where(condition) if condition is not None else stmt + + +def _admin_user_ids_stmt(): + return select(UserRow.id).where(UserRow.system_role == "admin") + + +def _scheduled_run_ids_stmt(): + return select(ScheduledTaskRunRow.agent_run_id).where(ScheduledTaskRunRow.agent_run_id.is_not(None)) + + +def _apply_user_scope(stmt, column, *, exclude_user_ids: set[str], include_admins: bool): + stmt = _apply_exclude(stmt, column, exclude_user_ids) + if not include_admins: + stmt = stmt.where(or_(column.is_(None), column.not_in(_admin_user_ids_stmt()))) + return stmt + + +def _apply_run_scope( + stmt, + *, + user_column, + run_id_column, + exclude_user_ids: set[str], + include_admins: bool, + include_failed: bool, + include_scheduled: bool, +): + stmt = _apply_user_scope(stmt, user_column, exclude_user_ids=exclude_user_ids, include_admins=include_admins) + if not include_failed: + stmt = stmt.where(RunRow.status != "error") + if not include_scheduled: + stmt = stmt.where(or_(run_id_column.is_(None), run_id_column.not_in(_scheduled_run_ids_stmt()))) + return stmt + + +def _apply_tool_scope( + stmt, + *, + exclude_user_ids: set[str], + include_admins: bool, + include_scheduled: bool, +): + stmt = _apply_user_scope(stmt, ToolCallMetricRow.user_id, exclude_user_ids=exclude_user_ids, include_admins=include_admins) + if not include_scheduled: + stmt = stmt.where(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt()))) + return stmt + + +async def _query_user_leaderboard( + session, + since: datetime, + until: datetime, + user_id: str | None = None, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardUser]: + exclude_user_ids = exclude_user_ids or set() run_agg = ( select( RunRow.user_id.label("user_id"), @@ -195,14 +284,27 @@ async def _query_user_leaderboard(session, since: datetime, until: datetime, use ) .where(_run_time_filter(since, until)) .group_by(RunRow.user_id) - .subquery() ) + run_agg = _apply_run_scope( + run_agg, + user_column=RunRow.user_id, + run_id_column=RunRow.run_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_failed=include_failed, + include_scheduled=include_scheduled, + ).subquery() thread_agg = ( select(ThreadMetaRow.user_id.label("user_id"), func.count(ThreadMetaRow.thread_id).label("thread_count")) .where(and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until)) .group_by(ThreadMetaRow.user_id) - .subquery() ) + thread_agg = _apply_user_scope( + thread_agg, + ThreadMetaRow.user_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + ).subquery() tool_agg = ( select( ToolCallMetricRow.user_id.label("user_id"), @@ -211,8 +313,13 @@ async def _query_user_leaderboard(session, since: datetime, until: datetime, use ) .where(_tool_time_filter(since, until)) .group_by(ToolCallMetricRow.user_id) - .subquery() ) + tool_agg = _apply_tool_scope( + tool_agg, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_scheduled=include_scheduled, + ).subquery() stmt = ( select( @@ -236,6 +343,7 @@ async def _query_user_leaderboard(session, since: datetime, until: datetime, use ) if user_id: stmt = stmt.where(UserRow.id == user_id) + stmt = _apply_user_scope(stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins) stmt = stmt.order_by( desc(func.coalesce(run_agg.c.run_count, 0) + func.coalesce(tool_agg.c.tool_call_count, 0)), desc(func.coalesce(run_agg.c.total_tokens, 0)), @@ -261,7 +369,19 @@ async def _query_user_leaderboard(session, since: datetime, until: datetime, use ] -async def _query_call_rankings(session, since: datetime, until: datetime, *, user_id: str | None, field: str, limit: int) -> list[AdminLeaderboardSkill]: +async def _query_call_rankings( + session, + since: datetime, + until: datetime, + *, + user_id: str | None, + field: str, + limit: int, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, +) -> list[AdminLeaderboardSkill]: + exclude_user_ids = exclude_user_ids or set() name_col = ToolCallMetricRow.skill_name if field == "skill" else ToolCallMetricRow.tool_name call_count = func.count(ToolCallMetricRow.id).label("call_count") last_used_at = func.max(ToolCallMetricRow.created_at).label("last_used_at") @@ -282,6 +402,12 @@ async def _query_call_rankings(session, since: datetime, until: datetime, *, use ) if user_id: stmt = stmt.where(ToolCallMetricRow.user_id == user_id) + stmt = _apply_tool_scope( + stmt, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_scheduled=include_scheduled, + ) rows = (await session.execute(stmt)).all() return [ AdminLeaderboardSkill( @@ -297,7 +423,142 @@ async def _query_call_rankings(session, since: datetime, until: datetime, *, use ] -async def _query_recent_questions(session, since: datetime, until: datetime, *, user_id: str | None, limit: int) -> list[AdminLeaderboardQuestion]: +async def _query_run_dimensions( + session, + since: datetime, + until: datetime, + *, + user_id: str | None, + field: str, + limit: int, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardDimension]: + exclude_user_ids = exclude_user_ids or set() + if field == "model": + name_col = RunRow.model_name + elif field == "agent": + name_col = RunRow.assistant_id + elif field == "status": + name_col = RunRow.status + else: + raise ValueError(f"Unknown run dimension: {field}") + + count = func.count(RunRow.run_id).label("count") + last_seen_at = func.max(RunRow.updated_at).label("last_seen_at") + stmt = ( + select( + name_col.label("name"), + count, + func.count(func.distinct(RunRow.user_id)).label("user_count"), + func.sum(case((RunRow.status == "success", 1), else_=0)).label("success_count"), + func.sum(case((RunRow.status == "error", 1), else_=0)).label("error_count"), + func.coalesce(func.sum(RunRow.total_tokens), 0).label("total_tokens"), + last_seen_at, + ) + .where(_run_time_filter(since, until), name_col.is_not(None), name_col != "") + .group_by(name_col) + .order_by(count.desc(), last_seen_at.desc()) + .limit(limit) + ) + if user_id: + stmt = stmt.where(RunRow.user_id == user_id) + stmt = _apply_run_scope( + stmt, + user_column=RunRow.user_id, + run_id_column=RunRow.run_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_failed=include_failed, + include_scheduled=include_scheduled, + ) + rows = (await session.execute(stmt)).all() + return [ + AdminLeaderboardDimension( + name=r[0] or "", + count=int(r[1] or 0), + user_count=int(r[2] or 0), + success_count=int(r[3] or 0), + error_count=int(r[4] or 0), + total_tokens=int(r[5] or 0), + last_seen_at=_iso(r[6]), + ) + for r in rows + ] + + +async def _query_model_rankings( + session, + since: datetime, + until: datetime, + *, + user_id: str | None, + limit: int, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardDimension]: + exclude_user_ids = exclude_user_ids or set() + count = func.count(LlmCallMetricRow.id).label("count") + last_seen_at = func.max(LlmCallMetricRow.created_at).label("last_seen_at") + stmt = ( + select( + LlmCallMetricRow.model_name.label("name"), + count, + func.count(func.distinct(LlmCallMetricRow.user_id)).label("user_count"), + func.sum(case((LlmCallMetricRow.status == "success", 1), else_=0)).label("success_count"), + func.sum(case((LlmCallMetricRow.status == "error", 1), else_=0)).label("error_count"), + func.coalesce(func.sum(LlmCallMetricRow.total_tokens), 0).label("total_tokens"), + last_seen_at, + ) + .where(_llm_time_filter(since, until), LlmCallMetricRow.model_name.is_not(None), LlmCallMetricRow.model_name != "") + .group_by(LlmCallMetricRow.model_name) + .order_by(count.desc(), last_seen_at.desc()) + .limit(limit) + ) + if user_id: + stmt = stmt.where(LlmCallMetricRow.user_id == user_id) + stmt = _apply_user_scope( + stmt, + LlmCallMetricRow.user_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + ) + if not include_failed: + stmt = stmt.where(LlmCallMetricRow.status != "error") + if not include_scheduled: + stmt = stmt.where(or_(LlmCallMetricRow.run_id.is_(None), LlmCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt()))) + rows = (await session.execute(stmt)).all() + return [ + AdminLeaderboardDimension( + name=r[0] or "", + count=int(r[1] or 0), + user_count=int(r[2] or 0), + success_count=int(r[3] or 0), + error_count=int(r[4] or 0), + total_tokens=int(r[5] or 0), + last_seen_at=_iso(r[6]), + ) + for r in rows + ] + + +async def _query_recent_questions( + session, + since: datetime, + until: datetime, + *, + user_id: str | None, + limit: int, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardQuestion]: + exclude_user_ids = exclude_user_ids or set() stmt = ( select(RunRow.first_human_message, RunRow.user_id, UserRow.email, RunRow.thread_id, RunRow.run_id, RunRow.created_at) .outerjoin(UserRow, UserRow.id == RunRow.user_id) @@ -307,6 +568,15 @@ async def _query_recent_questions(session, since: datetime, until: datetime, *, ) if user_id: stmt = stmt.where(RunRow.user_id == user_id) + stmt = _apply_run_scope( + stmt, + user_column=RunRow.user_id, + run_id_column=RunRow.run_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_failed=include_failed, + include_scheduled=include_scheduled, + ) rows = (await session.execute(stmt)).all() return [ AdminLeaderboardQuestion( @@ -321,7 +591,17 @@ async def _query_recent_questions(session, since: datetime, until: datetime, *, ] -async def _query_frequent_questions(session, since: datetime, until: datetime, limit: int) -> list[AdminLeaderboardQuestion]: +async def _query_frequent_questions( + session, + since: datetime, + until: datetime, + limit: int, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardQuestion]: + exclude_user_ids = exclude_user_ids or set() stmt = ( select( RunRow.first_human_message, @@ -337,6 +617,15 @@ async def _query_frequent_questions(session, since: datetime, until: datetime, l .order_by(desc("count"), desc("asked_at")) .limit(limit) ) + stmt = _apply_run_scope( + stmt, + user_column=RunRow.user_id, + run_id_column=RunRow.run_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_failed=include_failed, + include_scheduled=include_scheduled, + ) rows = (await session.execute(stmt)).all() return [ AdminLeaderboardQuestion( @@ -349,7 +638,18 @@ async def _query_frequent_questions(session, since: datetime, until: datetime, l ] -async def _query_trends(session, since: datetime, until: datetime, *, user_id: str | None = None) -> list[AdminLeaderboardTrendPoint]: +async def _query_trends( + session, + since: datetime, + until: datetime, + *, + user_id: str | None = None, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> list[AdminLeaderboardTrendPoint]: + exclude_user_ids = exclude_user_ids or set() bucket = func.date(RunRow.created_at) stmt = ( select( @@ -365,6 +665,15 @@ async def _query_trends(session, since: datetime, until: datetime, *, user_id: s ) if user_id: stmt = stmt.where(RunRow.user_id == user_id) + stmt = _apply_run_scope( + stmt, + user_column=RunRow.user_id, + run_id_column=RunRow.run_id, + exclude_user_ids=exclude_user_ids, + include_admins=include_admins, + include_failed=include_failed, + include_scheduled=include_scheduled, + ) rows = (await session.execute(stmt)).all() return [ AdminLeaderboardTrendPoint( @@ -378,7 +687,18 @@ async def _query_trends(session, since: datetime, until: datetime, *, user_id: s ] -async def _query_overview(session, since: datetime, until: datetime, *, user_id: str | None = None) -> AdminLeaderboardOverview: +async def _query_overview( + session, + since: datetime, + until: datetime, + *, + user_id: str | None = None, + exclude_user_ids: set[str] | None = None, + include_scheduled: bool = True, + include_admins: bool = True, + include_failed: bool = True, +) -> AdminLeaderboardOverview: + exclude_user_ids = exclude_user_ids or set() run_where = [_run_time_filter(since, until)] thread_where = [and_(ThreadMetaRow.created_at >= since, ThreadMetaRow.created_at <= until)] tool_where = [_tool_time_filter(since, until)] @@ -386,8 +706,23 @@ async def _query_overview(session, since: datetime, until: datetime, *, user_id: run_where.append(RunRow.user_id == user_id) thread_where.append(ThreadMetaRow.user_id == user_id) tool_where.append(ToolCallMetricRow.user_id == user_id) + if exclude_user_ids: + run_where.append(_exclude_user_filter(RunRow.user_id, exclude_user_ids)) + thread_where.append(_exclude_user_filter(ThreadMetaRow.user_id, exclude_user_ids)) + tool_where.append(_exclude_user_filter(ToolCallMetricRow.user_id, exclude_user_ids)) + if not include_admins: + run_where.append(or_(RunRow.user_id.is_(None), RunRow.user_id.not_in(_admin_user_ids_stmt()))) + thread_where.append(or_(ThreadMetaRow.user_id.is_(None), ThreadMetaRow.user_id.not_in(_admin_user_ids_stmt()))) + tool_where.append(or_(ToolCallMetricRow.user_id.is_(None), ToolCallMetricRow.user_id.not_in(_admin_user_ids_stmt()))) + if not include_failed: + run_where.append(RunRow.status != "error") + if not include_scheduled: + run_where.append(or_(RunRow.run_id.is_(None), RunRow.run_id.not_in(_scheduled_run_ids_stmt()))) + tool_where.append(or_(ToolCallMetricRow.run_id.is_(None), ToolCallMetricRow.run_id.not_in(_scheduled_run_ids_stmt()))) - total_users = 1 if user_id else (await session.execute(select(func.count(UserRow.id)))).scalar_one() + total_users_stmt = select(func.count(UserRow.id)) + total_users_stmt = _apply_user_scope(total_users_stmt, UserRow.id, exclude_user_ids=exclude_user_ids, include_admins=include_admins) + total_users = 1 if user_id else (await session.execute(total_users_stmt)).scalar_one() run_row = (await session.execute( select( func.count(RunRow.run_id), @@ -424,6 +759,10 @@ async def get_user_leaderboard( days: int = Query(default=30, ge=1, le=365), since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"), until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"), + exclude_user_ids: list[str] | None = Query(default=None, description="User ids excluded from this leaderboard aggregation"), + include_scheduled: bool = Query(default=True, description="Include runs created by scheduled tasks"), + include_admins: bool = Query(default=True, description="Include admin accounts in leaderboard aggregation"), + include_failed: bool = Query(default=True, description="Include failed runs and LLM calls"), limit: int = Query(default=20, ge=5, le=100), ) -> AdminLeaderboardResponse: """Return admin analytics for user activity, Q&A, skills, questions, and trends.""" @@ -433,14 +772,31 @@ async def get_user_leaderboard( raise HTTPException(status_code=503, detail="Analytics requires database persistence") since_dt, until_dt, range_days = _resolve_range(days, since, until) + excluded = {uid for uid in (exclude_user_ids or []) if uid} async with session_factory() as session: - overview = await _query_overview(session, since_dt, until_dt) - users = await _query_user_leaderboard(session, since_dt, until_dt) - skills = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="skill", limit=limit) - tools = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="tool", limit=limit) - recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=None, limit=limit) - frequent_questions = await _query_frequent_questions(session, since_dt, until_dt, limit=limit) - trends = await _query_trends(session, since_dt, until_dt) + common = { + "exclude_user_ids": excluded, + "include_scheduled": include_scheduled, + "include_admins": include_admins, + "include_failed": include_failed, + } + tool_common = { + "exclude_user_ids": excluded, + "include_scheduled": include_scheduled, + "include_admins": include_admins, + } + overview = await _query_overview(session, since_dt, until_dt, **common) + users = await _query_user_leaderboard(session, since_dt, until_dt, **common) + skills = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="skill", limit=limit, **tool_common) + tools = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="tool", limit=limit, **tool_common) + models = await _query_model_rankings(session, since_dt, until_dt, user_id=None, limit=limit, **common) + if not models: + models = await _query_run_dimensions(session, since_dt, until_dt, user_id=None, field="model", limit=limit, **common) + agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=None, field="agent", limit=limit, **common) + run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=None, field="status", limit=limit, **common) + recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=None, limit=limit, **common) + frequent_questions = await _query_frequent_questions(session, since_dt, until_dt, limit=limit, **common) + trends = await _query_trends(session, since_dt, until_dt, **common) return AdminLeaderboardResponse( range_days=range_days, @@ -452,6 +808,9 @@ async def get_user_leaderboard( users_by_tokens=sorted(users, key=lambda u: u.total_tokens, reverse=True)[:limit], skills=skills, tools=tools, + models=models, + agents=agents, + run_statuses=run_statuses, recent_questions=recent_questions, frequent_questions=frequent_questions, trends=trends, @@ -481,6 +840,11 @@ async def get_user_analytics( overview = await _query_overview(session, since_dt, until_dt, user_id=user_id) skills = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="skill", limit=limit) tools = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="tool", limit=limit) + models = await _query_model_rankings(session, since_dt, until_dt, user_id=user_id, limit=limit) + if not models: + models = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="model", limit=limit) + agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="agent", limit=limit) + run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="status", limit=limit) recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=user_id, limit=limit) trends = await _query_trends(session, since_dt, until_dt, user_id=user_id) @@ -492,6 +856,9 @@ async def get_user_analytics( overview=overview, skills=skills, tools=tools, + models=models, + agents=agents, + run_statuses=run_statuses, recent_questions=recent_questions, trends=trends, )