Merge branch ji into master: 工具/技能调用统计 + 用户活动审计 + 标签多选筛选

This commit is contained in:
吉鑫升 committed 2026-05-19 09:28:44 +08:00
commit bd1efd8026
46 files changed
+3299 -315

No files matched your search

@@ -131,6 +131,14 @@ def _build_runtime_middlewares(
middlewares.append(SandboxAuditMiddleware())
middlewares.append(ToolErrorHandlingMiddleware())
# Tool / skill call metrics — appended last so it sits *deepest* in the
# wrap_tool_call chain: its handler is the raw tool, so it observes the
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
# an error ToolMessage) and the raw return value.
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
middlewares.append(ToolMetricsMiddleware())
return middlewares
@@ -0,0 +1,319 @@
"""Middleware that records per-tool-call metrics for the admin audit page.
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
- Successes record the wall-clock duration.
- Failures record a coarse ``error_category`` plus a one-line message.
Two failure shapes are captured:
1. **Raised exceptions** — the tool blew up. We record, then re-raise so the
outer :class:`ToolErrorHandlingMiddleware` still converts it into an error
``ToolMessage`` for the model.
2. **Self-handled errors** — search/retrieval tools (Serper, Tavily, Jina, …)
catch their own HTTP 429/403/timeout and *return* an error payload instead
of raising. Those never reach an ``except`` block, so we also inspect the
returned ``ToolMessage``: an ``status == "error"`` message, or content that
looks like an error string, is bucketed too.
This middleware is appended **after** ``ToolErrorHandlingMiddleware`` so it
sits deepest in the ``wrap_tool_call`` chain — its handler is the raw tool, so
it observes the unconverted exception and the raw return value.
Writes go through a best-effort store: a DB outage logs a warning and the
agent runtime keeps running.
"""
from __future__ import annotations
import asyncio
import logging
import re
import time
import uuid
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langchain_core.messages import ToolMessage
from langgraph.errors import GraphBubbleUp
from langgraph.prebuilt.tool_node import ToolCallRequest
from langgraph.types import Command
from deerflow.persistence.engine import get_session_factory
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
def _now() -> datetime:
return datetime.now(UTC)
def _trim(text: str, limit: int = 500) -> str:
text = (text or "").strip()
return text if len(text) <= limit else text[: limit - 3] + "..."
def _classify_error(text: str) -> str:
"""Bucket an error string into a coarse, admin-friendly category."""
t = (text or "").lower()
if any(k in t for k in ("429", "rate limit", "ratelimit", "too many request", "quota")):
return "rate_limited"
if any(k in t for k in ("403", "forbidden", "ip ban", "ip block", "blocked", "access denied", "banned")):
return "ip_blocked"
if any(k in t for k in ("401", "unauthorized", "api key", "apikey", "invalid key", "authentication", "credential")):
return "auth_error"
if any(k in t for k in ("timeout", "timed out", "deadline")):
return "timeout"
if any(k in t for k in ("connecterror", "connection error", "connection refused", "network", "dns", "unreachable", "ssl", "proxy")):
return "network_error"
if any(k in t for k in ("404", "not found")):
return "not_found"
if any(k in t for k in ("no results", "no result", "0 results", "empty result")):
return "empty_result"
return "tool_error"
def _content_text(content: Any) -> str:
"""Flatten a tool result's content (string or block list) into text."""
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
return " ".join(parts)
return str(content)
def _looks_like_error(text: str) -> bool:
"""Heuristically detect a self-handled error payload returned as content.
Deliberately conservative — only an explicit ``Error:`` prefix or a JSON
object whose first key is ``error`` counts, so a normal result that merely
mentions the word "error" is not misclassified.
"""
head = (text or "").lstrip()[:64].lower()
return head.startswith("error:") or head.startswith('{"error"') or head.startswith("{'error'")
# Skill tools whose ``name`` argument names one specific Agent Skill directly.
_SKILL_NAME_ARG_TOOLS = {"skill_view", "skill_manage", "skill_archive"}
# Matches a skill directory embedded in any tool-argument path. Skills live
# under ``skills/{public,custom}/{name}/`` — both inside the sandbox
# (``/mnt/skills/...``) and on the host. The lead agent's system prompt tells
# it to load a skill by calling ``read_file`` on that SKILL.md path (it does
# NOT call a dedicated "use skill" tool), so the skill name has to be recovered
# from the path argument of whatever tool touched it (read_file / bash / ls /
# str_replace / write_file ...).
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
def _skill_name_from_call(tool_name: str | None, tool_call: Any) -> str | None:
"""Recover the Agent Skill a tool call targets, or ``None``.
Two detection paths:
1. ``skill_view`` / ``skill_manage`` / ``skill_archive`` take an explicit
``name`` argument.
2. Every other tool references a skill by *path*. The canonical skill
invocation is ``read_file('/mnt/skills/<cat>/<name>/SKILL.md')``; helper
files and ``bash`` script runs hit the same directory. So we scan the
string arguments for the ``skills/{public,custom}/{name}/`` shape.
"""
if not isinstance(tool_call, dict):
return None
args = tool_call.get("args")
if not isinstance(args, dict):
return None
# Case 1 — explicit ``name`` argument on the skill tools.
if tool_name in _SKILL_NAME_ARG_TOOLS:
raw = args.get("name")
if isinstance(raw, str) and raw.strip():
return raw.strip()
# Case 2 — a skill path inside any string argument.
for value in args.values():
if isinstance(value, str):
match = _SKILL_PATH_RE.search(value)
if match:
return match.group(1)
return None
def _evaluate_skill_view(text: str) -> tuple[str, str | None, str | None]:
"""Classify a ``skill_view`` return.
``skill_view`` never raises — on failure it returns a plain string
(``Skill 'x' not found.`` / ``Error reading skill 'x': ...``); on success
it returns the SKILL.md content. So success/failure must be read off the
returned text, not an exception or ``status`` flag.
"""
head = (text or "").lstrip()
low = head[:200].lower()
if head.startswith("Skill '") and "not found" in low:
return "error", "not_found", _trim(text)
if low.startswith("error reading skill"):
return "error", _classify_error(text), _trim(text)
return "success", None, None
def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | None, str | None]:
"""Return ``(status, error_category, error_message)`` for a tool return."""
if isinstance(result, ToolMessage):
text = _content_text(result.content)
if getattr(result, "status", None) == "error":
return "error", _classify_error(text), _trim(text)
# skill_view swallows its own failures into a plain string return.
if tool_name == "skill_view":
return _evaluate_skill_view(text)
if _looks_like_error(text):
return "error", _classify_error(text), _trim(text)
return "success", None, None
# Command (control flow) or any other shape — treat as a successful call.
return "success", None, None
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
"""Wrap each tool call to time it and persist a metric row."""
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
super().__init__(**kwargs)
self._store_override = store
# -- store / context ----------------------------------------------------
def _store(self) -> ToolMetricsStore | None:
if self._store_override is not None:
return self._store_override
try:
sf = get_session_factory()
except Exception:
return None
return make_tool_metrics_store(sf)
@staticmethod
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
runtime = getattr(request, "runtime", None)
if runtime is None:
return None
context = getattr(runtime, "context", None)
if isinstance(context, dict):
for key in keys:
value = context.get(key)
if isinstance(value, str) and value:
return value
config = getattr(runtime, "config", None)
if isinstance(config, dict):
configurable = config.get("configurable") if isinstance(config.get("configurable"), dict) else {}
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
for bucket in (configurable, metadata):
for key in keys:
value = bucket.get(key)
if isinstance(value, str) and value:
return value
return None
def _build_base_record(self, request: ToolCallRequest) -> ToolMetricRecord:
tool_call = getattr(request, "tool_call", None) or {}
tool_name = tool_call.get("name") if isinstance(tool_call, dict) else None
tool_name = str(tool_name) if tool_name else None
return ToolMetricRecord(
id=uuid.uuid4().hex,
created_at=_now(),
user_id=get_effective_user_id(),
thread_id=self._runtime_lookup(request, ("thread_id",)),
run_id=self._runtime_lookup(request, ("run_id", "langgraph_run_id")),
agent_name=self._runtime_lookup(request, ("agent_name", "agent_id")),
skill_name=_skill_name_from_call(tool_name, tool_call),
tool_name=tool_name,
)
# -- finalize -----------------------------------------------------------
def _finalize_result(self, record: ToolMetricRecord, result: Any, started: float) -> None:
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
record.status, record.error_category, record.error_message = _evaluate_result(
result, record.tool_name
)
def _finalize_error(self, record: ToolMetricRecord, exc: BaseException, started: float) -> None:
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
record.status = "error"
name = exc.__class__.__name__
combined = f"{name}: {(str(exc) or name).strip()}"
record.error_category = _classify_error(combined)
record.error_message = _trim(combined)
def _flush(self, record: ToolMetricRecord) -> None:
store = self._store()
if store is None:
return
try:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = None
if loop is not None and loop.is_running():
# Fire-and-forget — never let metric IO block the tool path.
loop.create_task(store.record(record))
else:
asyncio.run(store.record(record))
except Exception:
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
# -- wrap ---------------------------------------------------------------
@override
def wrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], ToolMessage | Command],
) -> ToolMessage | Command:
record = self._build_base_record(request)
started = time.monotonic()
try:
result = handler(request)
except GraphBubbleUp:
# LangGraph control flow — not a tool failure.
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_result(record, result, started)
self._flush(record)
return result
@override
async def awrap_tool_call(
self,
request: ToolCallRequest,
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
) -> ToolMessage | Command:
record = self._build_base_record(request)
started = time.monotonic()
try:
result = await handler(request)
except GraphBubbleUp:
raise
except Exception as exc:
self._finalize_error(record, exc, started)
self._flush(record)
raise
self._finalize_result(record, result, started)
self._flush(record)
return result
@@ -0,0 +1,58 @@
"""Create tool_call_metrics table.
Revision ID: 20260518_07
Revises: 20260518_06
Create Date: 2026-05-18
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260518_07"
down_revision = "20260518_06"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
if _has_table("tool_call_metrics"):
return
op.create_table(
"tool_call_metrics",
sa.Column("id", sa.String(64), primary_key=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("user_id", sa.String(64), nullable=True),
sa.Column("thread_id", sa.String(64), nullable=True),
sa.Column("run_id", sa.String(64), nullable=True),
sa.Column("agent_name", sa.String(128), nullable=True),
sa.Column("tool_name", sa.String(128), nullable=True),
sa.Column("skill_name", sa.String(128), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
sa.Column("error_category", sa.String(32), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_tool_call_metrics_created_at", "tool_call_metrics", ["created_at"])
op.create_index("ix_tool_call_metrics_user_id", "tool_call_metrics", ["user_id"])
op.create_index("ix_tool_call_metrics_thread_id", "tool_call_metrics", ["thread_id"])
op.create_index("ix_tool_call_metrics_run_id", "tool_call_metrics", ["run_id"])
op.create_index("ix_tool_call_metrics_agent_name", "tool_call_metrics", ["agent_name"])
op.create_index("ix_tool_call_metrics_tool_name", "tool_call_metrics", ["tool_name"])
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
op.create_index("ix_tool_call_metrics_status", "tool_call_metrics", ["status"])
op.create_index("ix_tool_call_metrics_error_category", "tool_call_metrics", ["error_category"])
def downgrade() -> None:
if _has_table("tool_call_metrics"):
op.drop_table("tool_call_metrics")
@@ -0,0 +1,62 @@
"""Add skill_name column to tool_call_metrics.
``tool_call_metrics`` is created by ``create_all`` at startup. On deployments
where the table was materialised *before* the ``skill_name`` column existed,
neither ``create_all`` (it never ALTERs) nor revision ``20260518_07`` (it skips
when the table already exists) adds the column — leaving MySQL/Postgres with a
schema that the ORM model expects a column it does not have. This revision
patches the existing table.
Revision ID: 20260519_01
Revises: 20260518_07
Create Date: 2026-05-19
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20260519_01"
down_revision = "20260518_07"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()
def _has_column(table_name: str, column_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
def _has_index(table_name: str, index_name: str) -> bool:
if not _has_table(table_name):
return False
inspector = inspect(op.get_bind())
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
def upgrade() -> None:
# Brand-new DBs: the table is created with the column already present
# (by create_all / revision 20260518_07) — nothing to patch.
if not _has_table("tool_call_metrics"):
return
if not _has_column("tool_call_metrics", "skill_name"):
op.add_column("tool_call_metrics", sa.Column("skill_name", sa.String(128), nullable=True))
if not _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
def downgrade() -> None:
if _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
op.drop_index("ix_tool_call_metrics_skill_name", table_name="tool_call_metrics")
if _has_column("tool_call_metrics", "skill_name"):
op.drop_column("tool_call_metrics", "skill_name")
@@ -33,6 +33,7 @@ from deerflow.persistence.skills.model import SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.user.model import UserRow
__all__ = [
@@ -53,5 +54,6 @@ __all__ = [
"TagRow",
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UserRow",
]
@@ -0,0 +1,27 @@
"""Persistence layer for per-tool-call (skill-call) metrics.
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
writes one row recording duration and outcome. Failures are bucketed into a
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
network_error / not_found / empty_result / tool_error) so administrators can
spot retrieval outages, blocked IPs, and quota problems at a glance.
"""
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
__all__ = [
"ToolCallMetricRow",
"ToolMetricRecord",
"ToolMetricsStore",
"SqlToolMetricsStore",
"make_tool_metrics_store",
]
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
if session_factory is None:
return None
return SqlToolMetricsStore(session_factory)
@@ -0,0 +1,84 @@
"""Abstract repository contract for tool / skill call metrics."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any
@dataclass
class ToolMetricRecord:
"""In-memory shape of a tool-call metric row before persistence."""
id: str
created_at: datetime
user_id: str | None = None
thread_id: str | None = None
run_id: str | None = None
agent_name: str | None = None
tool_name: str | None = None
# The specific Agent Skill this call targets, when the tool is a skill tool
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
# call's ``name`` argument. Null for every non-skill tool call. This is
# what lets the admin see "skill X was invoked, and succeeded/failed".
skill_name: str | None = None
duration_ms: int = 0
status: str = "success"
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
# network_error / not_found / empty_result / tool_error. Null on success.
error_category: str | None = None
error_message: str | None = None
@dataclass
class ToolMetricsQuery:
"""Filter set for the admin list / export endpoints."""
user_id: str | None = None
tool_name: str | None = None
# When True, keep only rows that target a specific Agent Skill
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
skill_only: bool = False
# Filter to one specific skill by name.
skill_name: str | None = None
status: str | None = None
error_category: str | None = None
since: datetime | None = None
until: datetime | None = None
limit: int = 100
offset: int = 0
class ToolMetricsStore(ABC):
"""Persistence contract for the tool-metrics middleware + admin API."""
@abstractmethod
async def record(self, metric: ToolMetricRecord) -> None:
"""Persist one metric row. Must never raise on best-effort writes."""
raise NotImplementedError
@abstractmethod
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Return the matching rows (newest first), paginated by ``query``."""
raise NotImplementedError
@abstractmethod
async def count(self, query: ToolMetricsQuery) -> int:
"""Return the unpaginated row count for ``query``."""
raise NotImplementedError
@abstractmethod
async def iter_all(self, query: ToolMetricsQuery):
"""Yield rows matching ``query`` without pagination — used by export."""
raise NotImplementedError
@abstractmethod
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
Only rows with a non-null ``skill_name`` are counted, so this answers
"which skills were invoked, and how often did each succeed/fail".
"""
raise NotImplementedError
@@ -0,0 +1,59 @@
"""ORM model for per-tool-call (skill-call) metrics."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ToolCallMetricRow(Base):
"""One row per completed (or failed) tool / skill invocation.
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
failures — search tools that swallow an HTTP 429/403 and return an error
payload instead of raising — so an admin can see "retrieval failed" /
"IP blocked" events that never surfaced as exceptions.
"""
__tablename__ = "tool_call_metrics"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
# The specific Agent Skill targeted by a skill tool call (from the call's
# ``name`` argument); null for non-skill tool calls.
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
# "success" or "error".
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
# Coarse failure bucket; null on success.
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
error_message: Mapped[str | None] = mapped_column(Text)
def to_dict(self) -> dict:
return {
"id": self.id,
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
"user_id": self.user_id,
"thread_id": self.thread_id,
"run_id": self.run_id,
"agent_name": self.agent_name,
"tool_name": self.tool_name,
"skill_name": self.skill_name,
"duration_ms": self.duration_ms,
"status": self.status,
"error_category": self.error_category,
"error_message": self.error_message,
}
@@ -0,0 +1,138 @@
"""SQLAlchemy-backed tool metrics repository."""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import desc, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
logger = logging.getLogger(__name__)
def _normalize_created_at(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
class SqlToolMetricsStore(ToolMetricsStore):
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, metric: ToolMetricRecord) -> None:
row = ToolCallMetricRow(
id=metric.id,
created_at=_normalize_created_at(metric.created_at),
user_id=metric.user_id,
thread_id=metric.thread_id,
run_id=metric.run_id,
agent_name=metric.agent_name,
tool_name=metric.tool_name,
skill_name=metric.skill_name,
duration_ms=metric.duration_ms,
status=metric.status,
error_category=metric.error_category,
error_message=metric.error_message,
)
try:
async with self._sf() as session:
session.add(row)
await session.commit()
except Exception:
# Metrics are best-effort; never surface a write failure to the runtime.
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
def _apply_filters(self, stmt, query: ToolMetricsQuery):
if query.user_id:
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
if query.tool_name:
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
if query.skill_only:
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
if query.skill_name:
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
if query.status:
stmt = stmt.where(ToolCallMetricRow.status == query.status)
if query.error_category:
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
if query.since is not None:
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
if query.until is not None:
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
return stmt
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
stmt = select(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
async with self._sf() as session:
result = await session.execute(stmt)
return [row.to_dict() for row in result.scalars()]
async def count(self, query: ToolMetricsQuery) -> int:
stmt = select(func.count()).select_from(ToolCallMetricRow)
stmt = self._apply_filters(stmt, query)
async with self._sf() as session:
return (await session.scalar(stmt)) or 0
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
page_size = 500
offset = 0
while True:
page = await self.list(
ToolMetricsQuery(
user_id=query.user_id,
tool_name=query.tool_name,
skill_only=query.skill_only,
skill_name=query.skill_name,
status=query.status,
error_category=query.error_category,
since=query.since,
until=query.until,
limit=page_size,
offset=offset,
)
)
if not page:
return
for item in page:
yield item
if len(page) < page_size:
return
offset += page_size
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
# Count success/error per skill. Force the skill-only filter so rows
# without a skill_name (plain tool calls) never leak into the result.
scoped = ToolMetricsQuery(
user_id=query.user_id,
skill_only=True,
skill_name=query.skill_name,
since=query.since,
until=query.until,
)
stmt = select(
ToolCallMetricRow.skill_name,
ToolCallMetricRow.status,
func.count().label("n"),
)
stmt = self._apply_filters(stmt, scoped).group_by(
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
)
agg: dict[str, dict[str, int]] = {}
async with self._sf() as session:
for skill_name, status, n in (await session.execute(stmt)).all():
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
bucket["total"] += n
bucket["success" if status == "success" else "error"] += n
return [
{"skill_name": name, **counts}
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
]