feat: 工具/技能调用统计 + 用户活动审计 + 标签多选筛选

- 任务1: 用户管理/定时任务管理移入左下角菜单, 从侧边栏删除原入口
- 任务2: 管理员查看用户问答(读 LangGraph checkpoint)与记忆, 新增 admin_users 路由与审计页
- 任务3: 标签筛选改为顶部多选标签栏(OR), agents/skills/scheduled_tasks 三端
- 任务4: 新增 tool_call_metrics 表 + ToolMetricsMiddleware + tool-metrics 路由,
  工具/技能调用统计并入大模型统计页, 含错误分类
- 技能识别: lead agent 经 read_file 加载 /mnt/skills/<cat>/<name>/SKILL.md 调用技能,
  中间件扫描工具参数里的 skills/{public,custom}/{name}/ 路径还原 skill_name
- 单元测试 tests/test_tool_metrics_middleware.py (18 项全过)
- 附: SQLite->MySQL 切换后清理脚本

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.7 committed 2026-05-19 00:59:15 +08:00
1 parent e3cde9be25
commit 4940e0d8ed
46 files changed
+3299 -315

No files matched your search

@@ -0,0 +1,58 @@
"""Create tool_call_metrics table.
Revision ID: 20260518_07
Revises: 20260518_06
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_07"
down_revision = "20260518_06"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("tool_call_metrics"):
return
op.create_table(
"tool_call_metrics",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("thread_id", sa.String(64), nullable=True),
sa.Column("run_id", sa.String(64), nullable=True),
sa.Column("agent_name", sa.String(128), nullable=True),
sa.Column("tool_name", sa.String(128), nullable=True),
sa.Column("skill_name", sa.String(128), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
sa.Column("error_category", sa.String(32), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_tool_call_metrics_created_at", "tool_call_metrics", ["created_at"])
op.create_index("ix_tool_call_metrics_user_id", "tool_call_metrics", ["user_id"])
op.create_index("ix_tool_call_metrics_thread_id", "tool_call_metrics", ["thread_id"])
op.create_index("ix_tool_call_metrics_run_id", "tool_call_metrics", ["run_id"])
op.create_index("ix_tool_call_metrics_agent_name", "tool_call_metrics", ["agent_name"])
op.create_index("ix_tool_call_metrics_tool_name", "tool_call_metrics", ["tool_name"])
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
op.create_index("ix_tool_call_metrics_status", "tool_call_metrics", ["status"])
op.create_index("ix_tool_call_metrics_error_category", "tool_call_metrics", ["error_category"])
def downgrade() -> None:
if _has_table("tool_call_metrics"):
op.drop_table("tool_call_metrics")
@@ -0,0 +1,62 @@
"""Add skill_name column to tool_call_metrics.
``tool_call_metrics`` is created by ``create_all`` at startup. On deployments
where the table was materialised *before* the ``skill_name`` column existed,
neither ``create_all`` (it never ALTERs) nor revision ``20260518_07`` (it skips
when the table already exists) adds the column — leaving MySQL/Postgres with a
schema that the ORM model expects a column it does not have. This revision
patches the existing table.
Revision ID: 20260519_01
Revises: 20260518_07
Create Date: 2026-05-19
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260519_01"
down_revision = "20260518_07"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
# Brand-new DBs: the table is created with the column already present
# (by create_all / revision 20260518_07) — nothing to patch.
if not _has_table("tool_call_metrics"):
return
if not _has_column("tool_call_metrics", "skill_name"):
op.add_column("tool_call_metrics", sa.Column("skill_name", sa.String(128), nullable=True))
if not _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
def downgrade() -> None:
if _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.drop_index("ix_tool_call_metrics_skill_name", table_name="tool_call_metrics")
if _has_column("tool_call_metrics", "skill_name"):
op.drop_column("tool_call_metrics", "skill_name")
@@ -33,6 +33,7 @@ from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
__all__ = [
@@ -53,5 +54,6 @@ __all__ = [
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UserRow",
]
@@ -0,0 +1,27 @@
"""Persistence layer for per-tool-call (skill-call) metrics.
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
writes one row recording duration and outcome. Failures are bucketed into a
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
network_error / not_found / empty_result / tool_error) so administrators can
spot retrieval outages, blocked IPs, and quota problems at a glance.
"""
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
__all__ = [
"ToolCallMetricRow",
"ToolMetricRecord",
"ToolMetricsStore",
"SqlToolMetricsStore",
"make_tool_metrics_store",
]
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlToolMetricsStore(session_factory)
@@ -0,0 +1,84 @@
"""Abstract repository contract for tool / skill call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class ToolMetricRecord:
"""In-memory shape of a tool-call metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
tool_name: str | None = None
# The specific Agent Skill this call targets, when the tool is a skill tool
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
# call's ``name`` argument. Null for every non-skill tool call. This is
# what lets the admin see "skill X was invoked, and succeeded/failed".
skill_name: str | None = None
duration_ms: int = 0
status: str = "success"
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
# network_error / not_found / empty_result / tool_error. Null on success.
error_category: str | None = None
error_message: str | None = None
@dataclass
class ToolMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
tool_name: str | None = None
# When True, keep only rows that target a specific Agent Skill
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
skill_only: bool = False
# Filter to one specific skill by name.
skill_name: str | None = None
status: str | None = None
error_category: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class ToolMetricsStore(ABC):
"""Persistence contract for the tool-metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: ToolMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: ToolMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: ToolMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@abstractmethod
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
Only rows with a non-null ``skill_name`` are counted, so this answers
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""ORM model for per-tool-call (skill-call) metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ToolCallMetricRow(Base):
"""One row per completed (or failed) tool / skill invocation.
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
failures — search tools that swallow an HTTP 429/403 and return an error
payload instead of raising — so an admin can see "retrieval failed" /
"IP blocked" events that never surfaced as exceptions.
"""
__tablename__ = "tool_call_metrics"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
# The specific Agent Skill targeted by a skill tool call (from the call's
# ``name`` argument); null for non-skill tool calls.
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
# "success" or "error".
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
# Coarse failure bucket; null on success.
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"tool_name": self.tool_name,
"skill_name": self.skill_name,
"duration_ms": self.duration_ms,
"status": self.status,
"error_category": self.error_category,
"error_message": self.error_message,
}
@@ -0,0 +1,138 @@
"""SQLAlchemy-backed tool metrics repository."""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
logger = logging.getLogger(__name__)
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
class SqlToolMetricsStore(ToolMetricsStore):
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: ToolMetricRecord) -> None:
row = ToolCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
tool_name=metric.tool_name,
skill_name=metric.skill_name,
duration_ms=metric.duration_ms,
status=metric.status,
error_category=metric.error_category,
error_message=metric.error_message,
)
try:
async with self._sf() as session:
session.add(row)
await session.commit()
except Exception:
# Metrics are best-effort; never surface a write failure to the runtime.
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: ToolMetricsQuery):
if query.user_id:
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
if query.tool_name:
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
if query.skill_only:
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
if query.skill_name:
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
if query.status:
stmt = stmt.where(ToolCallMetricRow.status == query.status)
if query.error_category:
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
if query.since is not None:
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
stmt = select(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [row.to_dict() for row in result.scalars()]
async def count(self, query: ToolMetricsQuery) -> int:
stmt = select(func.count()).select_from(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
page_size = 500
offset = 0
while True:
page = await self.list(
ToolMetricsQuery(
user_id=query.user_id,
tool_name=query.tool_name,
skill_only=query.skill_only,
skill_name=query.skill_name,
status=query.status,
error_category=query.error_category,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
# Count success/error per skill. Force the skill-only filter so rows
# without a skill_name (plain tool calls) never leak into the result.
scoped = ToolMetricsQuery(
user_id=query.user_id,
skill_only=True,
skill_name=query.skill_name,
since=query.since,
until=query.until,
)
stmt = select(
ToolCallMetricRow.skill_name,
ToolCallMetricRow.status,
func.count().label("n"),
)
stmt = self._apply_filters(stmt, scoped).group_by(
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
)
agg: dict[str, dict[str, int]] = {}
async with self._sf() as session:
for skill_name, status, n in (await session.execute(stmt)).all():
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
bucket["total"] += n
bucket["success" if status == "success" else "error"] += n
return [
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]