From a5897caec42a74c12d4faaeb019fcee14382313d Mon Sep 17 00:00:00 2001 From: Administrator Date: Fri, 5 Jun 2026 00:02:05 +0800 Subject: [PATCH] enhance leaderboard reporting and grouping --- frontend-web/src/core/admin/api.ts | 23 ++ frontend-web/src/core/admin/hooks.ts | 13 + .../src/pages/AdminUserLeaderboardPage.tsx | 373 ++++++++++-------- .../app/gateway/routers/admin_users.py | 159 +++++++- 4 files changed, 386 insertions(+), 182 deletions(-) diff --git a/frontend-web/src/core/admin/api.ts b/frontend-web/src/core/admin/api.ts index 0bafec09..725a02b9 100644 --- a/frontend-web/src/core/admin/api.ts +++ b/frontend-web/src/core/admin/api.ts @@ -133,6 +133,11 @@ export interface AdminUserAnalyticsResponse { trends: AdminLeaderboardTrendPoint[]; } +export interface AdminLeaderboardReportResponse { + report: string; + generated_at: string; +} + const base = () => `${getBackendBaseURL()}/api/admin/users`; async function parse(res: Response, what: string): Promise { @@ -208,6 +213,24 @@ export async function getUserAnalytics( ); } +export async function generateUserLeaderboardReport( + range: { since: string; until: string; days?: number }, + limit = 30, +): Promise { + const params = new URLSearchParams({ + days: String(range.days ?? 30), + since: range.since, + until: range.until, + limit: String(limit), + }); + return parse( + await apiFetch(`${base()}/leaderboard/report?${params}`, { + method: "POST", + }), + "Failed to generate leaderboard report", + ); +} + export async function addAdminMemoryEntry( userId: string, content: string, diff --git a/frontend-web/src/core/admin/hooks.ts b/frontend-web/src/core/admin/hooks.ts index fe2d7958..f08aec27 100644 --- a/frontend-web/src/core/admin/hooks.ts +++ b/frontend-web/src/core/admin/hooks.ts @@ -5,6 +5,7 @@ import { addAdminV1Fact, deleteAdminMemoryEntry, deleteAdminV1Fact, + generateUserLeaderboardReport, getUserMemory, getUserAnalytics, getUserLeaderboard, @@ -47,6 +48,18 @@ export function useUserAnalytics( }); } +export function useGenerateUserLeaderboardReport() { + return useMutation({ + mutationFn: ({ + range, + limit, + }: { + range: { since: string; until: string; days?: number }; + limit?: number; + }) => generateUserLeaderboardReport(range, limit ?? 30), + }); +} + /** Threads owned by a given user. Disabled until a user is picked. */ export function useUserThreads(userId: string | null) { return useQuery({ diff --git a/frontend-web/src/pages/AdminUserLeaderboardPage.tsx b/frontend-web/src/pages/AdminUserLeaderboardPage.tsx index 8b298575..ed15d687 100644 --- a/frontend-web/src/pages/AdminUserLeaderboardPage.tsx +++ b/frontend-web/src/pages/AdminUserLeaderboardPage.tsx @@ -40,7 +40,7 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { ScrollArea } from "@/components/ui/scroll-area"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { AdminLeaderboardDimension, AdminLeaderboardOverview, @@ -49,7 +49,7 @@ import type { AdminLeaderboardTrendPoint, AdminLeaderboardUser, } from "@/core/admin/api"; -import { useUserAnalytics, useUserLeaderboard } from "@/core/admin/hooks"; +import { useGenerateUserLeaderboardReport, useUserAnalytics, useUserLeaderboard } from "@/core/admin/hooks"; import { cn } from "@/lib/utils"; type UserRankTab = "activity" | "questions" | "tokens" | "skills" | "efficiency"; @@ -189,37 +189,36 @@ function exportUsersCsv(users: AdminLeaderboardUser[], tab: UserRankTab) { ); } -function exportReport(data: { - overview: AdminLeaderboardOverview; - users: AdminLeaderboardUser[]; - skills: AdminLeaderboardSkill[]; - models: AdminLeaderboardDimension[]; - questions: AdminLeaderboardQuestion[]; - range: RangeState; -}) { - const lines = [ - `用户排行榜统计报告`, - `时间范围：${data.range.since} 至 ${data.range.until}`, - ``, - `总用户：${fmtNumber(data.overview.total_users)}`, - `活跃用户：${fmtNumber(data.overview.active_users)}`, - `使用次数：${fmtNumber(data.overview.total_runs)}`, - `问答次数：${fmtNumber(data.overview.total_questions)}`, - `Token：${fmtNumber(data.overview.total_tokens)}`, - ``, - `Top 用户`, - ...data.users.slice(0, 10).map((user, index) => `${index + 1}. ${user.email || user.user_id} - 使用 ${fmtNumber(user.run_count)} / 问答 ${fmtNumber(user.qa_count)} / Token ${fmtNumber(user.total_tokens)}`), - ``, - `Top 技能`, - ...data.skills.slice(0, 10).map((item, index) => `${index + 1}. ${item.name} - ${fmtNumber(item.call_count)} 次`), - ``, - `Top 模型`, - ...data.models.slice(0, 10).map((item, index) => `${index + 1}. ${item.name} - ${fmtNumber(item.count)} 次 / Token ${fmtNumber(item.total_tokens)}`), - ``, - `近期问题`, - ...data.questions.slice(0, 10).map((item, index) => `${index + 1}. ${item.text}`), - ]; - downloadText(`user-leaderboard-report-${new Date().toISOString().slice(0, 10)}.txt`, lines.join("\n")); +function parseGroupRules(value: string): string[] { + return value + .split(/[\n,，;；\s]+/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function userGroupKey(user: AdminLeaderboardUser, rules: string[]): string { + const source = `${user.email || ""} ${user.user_id || ""}`.toLowerCase(); + const matched = rules.find((rule) => source.includes(rule.toLowerCase())); + return matched || "其他"; +} + +function buildGroupStats(users: AdminLeaderboardUser[], rules: string[]) { + const groups = new Map(); + for (const user of users) { + const key = userGroupKey(user, rules); + groups.set(key, [...(groups.get(key) || []), user]); + } + return [...groups.entries()] + .map(([name, members]) => ({ + name, + members: members.sort((a, b) => userScore(b, "activity") - userScore(a, "activity")), + userCount: members.length, + runs: members.reduce((sum, user) => sum + user.run_count, 0), + questions: members.reduce((sum, user) => sum + user.qa_count, 0), + tokens: members.reduce((sum, user) => sum + user.total_tokens, 0), + skills: members.reduce((sum, user) => sum + user.skill_call_count, 0), + })) + .sort((a, b) => b.runs + b.questions - (a.runs + a.questions)); } function initials(user: AdminLeaderboardUser): string { @@ -282,6 +281,73 @@ function InsightStrip({ overview }: { overview: AdminLeaderboardOverview }) { ); } +function GroupComparison({ + users, + rulesText, + onRulesChange, + onSelectUser, +}: { + users: AdminLeaderboardUser[]; + rulesText: string; + onRulesChange: (value: string) => void; + onSelectUser: (user: AdminLeaderboardUser) => void; +}) { + const rules = parseGroupRules(rulesText); + const groups = buildGroupStats(users, rules); + + return ( +
+
+
+
+ + 分组统计 +
+

+ 输入账号前缀或关键字，用逗号分隔，例如 zero,two；未命中的用户归入其他。 +

+
+ onRulesChange(event.target.value)} + placeholder="zero,two" + className="h-9 w-72 border-[#F1D6D6] bg-white dark:bg-[#19090A]" + /> +
+
+ {groups.map((group) => ( +
+
+
{group.name}
+ {group.userCount} 人 +
+
+ + + + +
+
+ {group.members.slice(0, 5).map((user, index) => ( + + ))} +
+
+ ))} +
+
+ ); +} + function PodiumCard({ user, rank, @@ -342,101 +408,6 @@ function PodiumCard({ ); } -function SpotlightCarousel({ - users, - tab, - onSelect, -}: { - users: AdminLeaderboardUser[]; - tab: UserRankTab; - onSelect: (user: AdminLeaderboardUser) => void; -}) { - const [index, setIndex] = useState(0); - const visible = users.slice(0, 5); - const active = visible[index % Math.max(visible.length, 1)]; - - useEffect(() => { - if (visible.length <= 1) return; - const timer = window.setInterval(() => { - setIndex((value) => (value + 1) % visible.length); - }, 5000); - return () => window.clearInterval(timer); - }, [visible.length]); - - useEffect(() => { - if (index >= visible.length) setIndex(0); - }, [index, visible.length]); - - if (!active) { - return ( -
-
暂无可轮播用户
-
- ); - } - - const maxScore = Math.max(1, ...visible.map((user) => userScore(user, tab))); - - return ( -
-
-
- - 榜单焦点 -
- 自动轮播 Top 5 -
-
- -
- {visible.map((user, itemIndex) => ( - - ))} -
-
-
- ); -} - function TrendBars({ points }: { points: AdminLeaderboardTrendPoint[] }) { type TrendMetric = keyof Pick; const [metric, setMetric] = useState<"all" | TrendMetric>("all"); @@ -910,20 +881,34 @@ function UserDetailPanel({ -
-
- - 个人维度 -
-
- - - -
-
- - - + + + 技能 + 提问 + 趋势 + 维度 + + + + + + + + +
+ + 个人使用趋势 +
+ + + +
+ + + +
+ + ); @@ -934,12 +919,15 @@ export default function AdminUserLeaderboardPage() { const [range, setRange] = useState(() => defaultRange(30)); const [tab, setTab] = useState("activity"); const [query, setQuery] = useState(""); + const [groupRules, setGroupRules] = useState("zero,two"); const [selectedUserId, setSelectedUserId] = useState(null); const apiRange = useMemo( () => rangeForApi(range), [range], ); - const { data, isLoading, error } = useUserLeaderboard(apiRange, 30); + const { data, isLoading, error } = useUserLeaderboard(apiRange, 100); + const reportMutation = useGenerateUserLeaderboardReport(); + const [reportOpen, setReportOpen] = useState(false); const rankedUsers = useMemo(() => { const base = data?.users_by_activity ?? []; @@ -1006,10 +994,11 @@ export default function AdminUserLeaderboardPage() { variant="outline" size="sm" onClick={() => navigate("/page/strategy/admin/user-leaderboard/settings")} - className="h-9" + className="h-9 w-9 px-0" + title="排行榜设置" + aria-label="排行榜设置" > - - 排行榜设置 + - + + + + + + + AI 榜单分析报告 + + 基于当前时间段的问答、用户、技能、模型和趋势统计生成运营分析。 + + + + {reportMutation.isPending && ( +
+ + 正在调用大模型生成报告 +
+ )} + {reportMutation.error && ( +
+ 生成失败：{reportMutation.error.message} +
+ )} + {reportMutation.data?.report && ( + 
+                        {reportMutation.data.report}
+
+ )} + + {reportMutation.data?.report && ( +
+ +
+ )} + +
event.currentTarget.showPicker?.()} + onFocus={(event) => event.currentTarget.showPicker?.()} onChange={(event) => setRange((prev) => ({ ...prev, since: event.target.value }))} className="h-9 w-36 bg-white dark:bg-[#0B2A50]" /> @@ -1055,6 +1087,8 @@ export default function AdminUserLeaderboardPage() { event.currentTarget.showPicker?.()} + onFocus={(event) => event.currentTarget.showPicker?.()} onChange={(event) => setRange((prev) => ({ ...prev, until: event.target.value }))} className="h-9 w-36 bg-white dark:bg-[#0B2A50]" /> @@ -1097,10 +1131,11 @@ export default function AdminUserLeaderboardPage() {
- setSelectedUserId(user.user_id)} + rulesText={groupRules} + onRulesChange={setGroupRules} + onSelectUser={(user) => setSelectedUserId(user.user_id)} />
diff --git a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py index bd94a5e3..ec92e8fc 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/admin_users.py +++ b/offline-backend-20260512/backend/app/gateway/routers/admin_users.py @@ -13,14 +13,17 @@ methods' ``user_id`` parameter is designed for. from __future__ import annotations +import json import logging import re +from difflib import SequenceMatcher from datetime import datetime, timedelta from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field from sqlalchemy import and_, case, desc, func, or_, select +from langchain_core.messages import HumanMessage, SystemMessage from app.gateway.deps import ( get_agent_store, @@ -31,6 +34,7 @@ from app.gateway.deps import ( from deerflow.agents.memory import BuiltinFileProvider from deerflow.config.paths import get_paths from deerflow.config.system_settings import load_system_settings +from deerflow.models import create_chat_model from deerflow.runtime import serialize_channel_values from app.gateway.routers.memory import _build_v1_provider, _is_v1 from deerflow.persistence.engine import get_session_factory @@ -158,6 +162,11 @@ class AdminUserAnalyticsResponse(BaseModel): trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list) +class AdminLeaderboardReportResponse(BaseModel): + report: str + generated_at: str + + def _now() -> datetime: return datetime.now() @@ -215,6 +224,24 @@ def _is_displayable_question(value: str | None) -> bool: return lowered not in {"uploaded_files", "attachments", "files"} +def _question_similarity_key(value: str) -> str: + return re.sub(r"[\W_]+", "", value.lower()) + + +def _message_content_to_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + parts.append(str(item.get("text") or item.get("content") or "")) + else: + parts.append(str(item)) + return "\n".join(part for part in parts if part.strip()) + return str(content or "") + + def _run_time_filter(since: datetime, until: datetime): return and_(RunRow.created_at >= since, RunRow.created_at <= until) @@ -640,17 +667,14 @@ async def _query_frequent_questions( stmt = ( select( RunRow.first_human_message, - func.count(RunRow.run_id).label("count"), - func.max(RunRow.user_id).label("user_id"), - func.max(UserRow.email).label("email"), - func.max(RunRow.created_at).label("asked_at"), + RunRow.user_id, + UserRow.email, + RunRow.created_at, ) .outerjoin(UserRow, UserRow.id == RunRow.user_id) .where(_run_time_filter(since, until), RunRow.first_human_message.is_not(None), RunRow.first_human_message != "") - .group_by(RunRow.first_human_message) - .having(func.count(RunRow.run_id) > 1) - .order_by(desc("count"), desc("asked_at")) - .limit(limit) + .order_by(RunRow.created_at.desc()) + .limit(min(max(limit * 20, 200), 1000)) ) stmt = _apply_run_scope( stmt, @@ -662,17 +686,39 @@ async def _query_frequent_questions( include_scheduled=include_scheduled, ) rows = (await session.execute(stmt)).all() - questions: list[AdminLeaderboardQuestion] = [] + groups: list[dict] = [] for r in rows: text = _clean_question_text(r[0]) if cleanup_system_questions else str(r[0] or "").strip() if cleanup_system_questions and not _is_displayable_question(text): continue + key = _question_similarity_key(text) + if not key: + continue + target = None + for group in groups: + score = SequenceMatcher(None, key, group["key"]).ratio() + if score >= 0.72 or key in group["key"] or group["key"] in key: + target = group + break + if target is None: + groups.append({"key": key, "text": text, "count": 1, "user_id": r[1] or "", "email": r[2] or "", "asked_at": r[3]}) + else: + target["count"] += 1 + if r[3] and (target.get("asked_at") is None or r[3] > target["asked_at"]): + target.update({"text": text, "user_id": r[1] or "", "email": r[2] or "", "asked_at": r[3]}) + + groups.sort(key=lambda item: (item["count"], _iso(item.get("asked_at"))), reverse=True) + questions: list[AdminLeaderboardQuestion] = [] + for group in groups[:limit]: + text = group["text"][:460] + if group["count"] > 1: + text = f"{text}（相似问题 {group['count']} 次）" questions.append( AdminLeaderboardQuestion( - text=text[:500], - user_id=r[2] or "", - email=r[3] or "", - asked_at=_iso(r[4]), + text=text, + user_id=group["user_id"], + email=group["email"], + asked_at=_iso(group.get("asked_at")), ) ) return questions @@ -866,6 +912,93 @@ async def get_user_leaderboard( ) +@router.post("/leaderboard/report", response_model=AdminLeaderboardReportResponse) +async def generate_user_leaderboard_report( + request: Request, + days: int = Query(default=30, ge=1, le=365), + since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"), + until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"), + limit: int = Query(default=30, ge=10, le=100), +) -> AdminLeaderboardReportResponse: + """Generate an LLM-written admin report for the current leaderboard window.""" + await _require_admin(request) + session_factory = get_session_factory() + if session_factory is None: + raise HTTPException(status_code=503, detail="Analytics requires database persistence") + + settings = load_system_settings().leaderboard + since_dt, until_dt, _range_days = _resolve_range(days, since, until) + excluded = {uid for uid in settings.exclude_user_ids if uid} + common = { + "exclude_user_ids": excluded, + "include_scheduled": settings.include_scheduled, + "include_admins": settings.include_admins, + "include_failed": settings.include_failed, + } + tool_common = { + "exclude_user_ids": excluded, + "include_scheduled": settings.include_scheduled, + "include_admins": settings.include_admins, + } + question_common = { + **common, + "cleanup_system_questions": settings.cleanup_system_questions, + } + + async with session_factory() as session: + overview = await _query_overview(session, since_dt, until_dt, **common) + users = await _query_user_leaderboard(session, since_dt, until_dt, **common) + skills = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="skill", limit=limit, **tool_common) + tools = await _query_call_rankings(session, since_dt, until_dt, user_id=None, field="tool", limit=limit, **tool_common) + models = await _query_model_rankings(session, since_dt, until_dt, user_id=None, limit=limit, **common) + recent_questions = await _query_recent_questions(session, since_dt, until_dt, user_id=None, limit=limit, **question_common) + frequent_questions = await _query_frequent_questions(session, since_dt, until_dt, limit=limit, **question_common) + trends = await _query_trends(session, since_dt, until_dt, **common) + + top_users = sorted(users, key=lambda u: (u.run_count + u.tool_call_count, u.total_tokens), reverse=True)[:12] + context = { + "range": {"since": _iso(since_dt), "until": _iso(until_dt)}, + "overview": overview.model_dump(), + "top_users": [u.model_dump() for u in top_users], + "top_skills": [s.model_dump() for s in skills[:12]], + "top_tools": [t.model_dump() for t in tools[:12]], + "top_models": [m.model_dump() for m in models[:12]], + "frequent_questions": [q.model_dump() for q in frequent_questions[:20]], + "recent_questions": [q.model_dump() for q in recent_questions[:30]], + "trends": [t.model_dump() for t in trends[-14:]], + } + prompt = ( + "请基于以下管理员排行榜统计数据，生成一份中文运营分析报告。" + "报告需要有结论，不要只罗列数字。必须分析：" + "1) 谁使用次数最多、问答最多、Token 消耗最高；" + "2) 用户通常在问什么、相似问题聚类反映了哪些关注方向；" + "3) 当前所有用户关注的事件/主题/业务方向；" + "4) 从问题重复、失败率、追问密度等信号推断用户是否满意系统回答，注明这是推断；" + "5) 哪些技能/工具/模型最常用，可能说明什么；" + "6) 给管理员 3-5 条可执行建议。" + "输出结构：摘要、关键发现、用户与分组观察、问题主题、满意度推断、建议。" + "\n\n统计数据如下：\n" + f"{json.dumps(context, ensure_ascii=False, default=str)}" + ) + try: + model = create_chat_model(name=None, thinking_enabled=False) + response = await model.ainvoke( + [ + SystemMessage(content="你是企业 AI 系统的运营分析师，擅长从用户行为、问题文本和调用指标中生成管理报告。"), + HumanMessage(content=prompt), + ], + config={"run_name": "admin_user_leaderboard_report"}, + ) + except Exception as exc: + logger.exception("Failed to generate leaderboard report") + raise HTTPException(status_code=503, detail=f"Report generation failed: {exc}") from exc + + return AdminLeaderboardReportResponse( + report=_message_content_to_text(getattr(response, "content", response)).strip(), + generated_at=_iso(_now()), + ) + + @router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse) async def get_user_analytics( user_id: str,