feat: 工具/技能调用统计 + 用户活动审计 + 标签多选筛选

- 任务1: 用户管理/定时任务管理移入左下角菜单, 从侧边栏删除原入口
- 任务2: 管理员查看用户问答(读 LangGraph checkpoint)与记忆, 新增 admin_users 路由与审计页
- 任务3: 标签筛选改为顶部多选标签栏(OR), agents/skills/scheduled_tasks 三端
- 任务4: 新增 tool_call_metrics 表 + ToolMetricsMiddleware + tool-metrics 路由,
  工具/技能调用统计并入大模型统计页, 含错误分类
- 技能识别: lead agent 经 read_file 加载 /mnt/skills/<cat>/<name>/SKILL.md 调用技能,
  中间件扫描工具参数里的 skills/{public,custom}/{name}/ 路径还原 skill_name
- 单元测试 tests/test_tool_metrics_middleware.py (18 项全过)
- 附: SQLite->MySQL 切换后清理脚本

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.7 committed 2026-05-19 00:59:15 +08:00
1 parent e3cde9be25
commit 4940e0d8ed
46 files changed
+3299 -315

No files matched your search

@@ -0,0 +1,27 @@
"""Persistence layer for per-tool-call (skill-call) metrics.
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
writes one row recording duration and outcome. Failures are bucketed into a
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
network_error / not_found / empty_result / tool_error) so administrators can
spot retrieval outages, blocked IPs, and quota problems at a glance.
"""
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
__all__ = [
"ToolCallMetricRow",
"ToolMetricRecord",
"ToolMetricsStore",
"SqlToolMetricsStore",
"make_tool_metrics_store",
]
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlToolMetricsStore(session_factory)
@@ -0,0 +1,84 @@
"""Abstract repository contract for tool / skill call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class ToolMetricRecord:
"""In-memory shape of a tool-call metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
tool_name: str | None = None
# The specific Agent Skill this call targets, when the tool is a skill tool
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
# call's ``name`` argument. Null for every non-skill tool call. This is
# what lets the admin see "skill X was invoked, and succeeded/failed".
skill_name: str | None = None
duration_ms: int = 0
status: str = "success"
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
# network_error / not_found / empty_result / tool_error. Null on success.
error_category: str | None = None
error_message: str | None = None
@dataclass
class ToolMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
tool_name: str | None = None
# When True, keep only rows that target a specific Agent Skill
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
skill_only: bool = False
# Filter to one specific skill by name.
skill_name: str | None = None
status: str | None = None
error_category: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class ToolMetricsStore(ABC):
"""Persistence contract for the tool-metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: ToolMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: ToolMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: ToolMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@abstractmethod
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
Only rows with a non-null ``skill_name`` are counted, so this answers
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""ORM model for per-tool-call (skill-call) metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ToolCallMetricRow(Base):
"""One row per completed (or failed) tool / skill invocation.
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
failures — search tools that swallow an HTTP 429/403 and return an error
payload instead of raising — so an admin can see "retrieval failed" /
"IP blocked" events that never surfaced as exceptions.
"""
__tablename__ = "tool_call_metrics"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
# The specific Agent Skill targeted by a skill tool call (from the call's
# ``name`` argument); null for non-skill tool calls.
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
# "success" or "error".
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
# Coarse failure bucket; null on success.
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"tool_name": self.tool_name,
"skill_name": self.skill_name,
"duration_ms": self.duration_ms,
"status": self.status,
"error_category": self.error_category,
"error_message": self.error_message,
}
@@ -0,0 +1,138 @@
"""SQLAlchemy-backed tool metrics repository."""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
logger = logging.getLogger(__name__)
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
class SqlToolMetricsStore(ToolMetricsStore):
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: ToolMetricRecord) -> None:
row = ToolCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
tool_name=metric.tool_name,
skill_name=metric.skill_name,
duration_ms=metric.duration_ms,
status=metric.status,
error_category=metric.error_category,
error_message=metric.error_message,
)
try:
async with self._sf() as session:
session.add(row)
await session.commit()
except Exception:
# Metrics are best-effort; never surface a write failure to the runtime.
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: ToolMetricsQuery):
if query.user_id:
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
if query.tool_name:
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
if query.skill_only:
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
if query.skill_name:
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
if query.status:
stmt = stmt.where(ToolCallMetricRow.status == query.status)
if query.error_category:
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
if query.since is not None:
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
stmt = select(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [row.to_dict() for row in result.scalars()]
async def count(self, query: ToolMetricsQuery) -> int:
stmt = select(func.count()).select_from(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
page_size = 500
offset = 0
while True:
page = await self.list(
ToolMetricsQuery(
user_id=query.user_id,
tool_name=query.tool_name,
skill_only=query.skill_only,
skill_name=query.skill_name,
status=query.status,
error_category=query.error_category,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
# Count success/error per skill. Force the skill-only filter so rows
# without a skill_name (plain tool calls) never leak into the result.
scoped = ToolMetricsQuery(
user_id=query.user_id,
skill_only=True,
skill_name=query.skill_name,
since=query.since,
until=query.until,
)
stmt = select(
ToolCallMetricRow.skill_name,
ToolCallMetricRow.status,
func.count().label("n"),
)
stmt = self._apply_filters(stmt, scoped).group_by(
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
)
agg: dict[str, dict[str, int]] = {}
async with self._sf() as session:
for skill_name, status, n in (await session.execute(stmt)).all():
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
bucket["total"] += n
bucket["success" if status == "success" else "error"] += n
return [
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]