初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+35
@@ -0,0 +1,35 @@
|
||||
"""Persistence layer for per-tool-call (skill-call) metrics.
|
||||
|
||||
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
|
||||
writes one row recording duration and outcome. Failures are bucketed into a
|
||||
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
|
||||
network_error / not_found / empty_result / tool_error) so administrators can
|
||||
spot retrieval outages, blocked IPs, and quota problems at a glance.
|
||||
|
||||
**Skill-call merging**: a skill is often touched by several tool calls inside
|
||||
one run (read ``SKILL.md``, run a script, retry after a failure). Rather than
|
||||
one row per call, ``SqlToolMetricsStore.record`` upserts a single row per
|
||||
``(run_id, skill_name)``: any success in the run wins (a skill that failed then
|
||||
finally succeeded is recorded as one success), and a skill that only ever
|
||||
failed yields one ``error`` row whose ``error_message`` accumulates every
|
||||
attempt's message. Non-skill tool calls keep one row each.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"ToolCallMetricRow",
|
||||
"ToolMetricRecord",
|
||||
"ToolMetricsStore",
|
||||
"SqlToolMetricsStore",
|
||||
"make_tool_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlToolMetricsStore(session_factory)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
"""Abstract repository contract for tool / skill call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricRecord:
|
||||
"""In-memory shape of a tool-call metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
tool_name: str | None = None
|
||||
# The specific Agent Skill this call targets, when the tool is a skill tool
|
||||
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
|
||||
# call's ``name`` argument. Null for every non-skill tool call. This is
|
||||
# what lets the admin see "skill X was invoked, and succeeded/failed".
|
||||
skill_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
status: str = "success"
|
||||
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
|
||||
# network_error / not_found / empty_result / tool_error. Null on success.
|
||||
error_category: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
tool_name: str | None = None
|
||||
# When True, keep only rows that target a specific Agent Skill
|
||||
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
|
||||
skill_only: bool = False
|
||||
# Filter to one specific skill by name.
|
||||
skill_name: str | None = None
|
||||
status: str | None = None
|
||||
error_category: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class ToolMetricsStore(ABC):
|
||||
"""Persistence contract for the tool-metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: ToolMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
|
||||
|
||||
Only rows with a non-null ``skill_name`` are counted, so this answers
|
||||
"which skills were invoked, and how often did each succeed/fail".
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""ORM model for per-tool-call (skill-call) metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ToolCallMetricRow(Base):
|
||||
"""One row per completed (or failed) tool / skill invocation.
|
||||
|
||||
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
|
||||
failures — search tools that swallow an HTTP 429/403 and return an error
|
||||
payload instead of raising — so an admin can see "retrieval failed" /
|
||||
"IP blocked" events that never surfaced as exceptions.
|
||||
"""
|
||||
|
||||
__tablename__ = "tool_call_metrics"
|
||||
__table_args__ = (
|
||||
Index("ix_tool_call_metrics_created_user", "created_at", "user_id"),
|
||||
Index("ix_tool_call_metrics_run_created", "run_id", "created_at"),
|
||||
Index("ix_tool_call_metrics_created_skill_user", "created_at", "skill_name", "user_id"),
|
||||
Index("ix_tool_call_metrics_created_tool_user", "created_at", "tool_name", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# The specific Agent Skill targeted by a skill tool call (from the call's
|
||||
# ``name`` argument); null for non-skill tool calls.
|
||||
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
# "success" or "error".
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
# Coarse failure bucket; null on success.
|
||||
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"tool_name": self.tool_name,
|
||||
"skill_name": self.skill_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"status": self.status,
|
||||
"error_category": self.error_category,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
"""SQLAlchemy-backed tool metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Separator joining every failed attempt's message inside one merged skill row.
|
||||
_ERROR_SEP = "\n--- \n"
|
||||
# Hard cap on the accumulated ``error_message`` of a merged row.
|
||||
_MERGED_ERROR_LIMIT = 2000
|
||||
|
||||
# A single skill invocation often spans several tool calls (read SKILL.md, run
|
||||
# a script, ...), and the model may retry a skill within the same run. Those
|
||||
# fire-and-forget writes can interleave on the event loop, so the read → merge →
|
||||
# write of a skill row is guarded by a lock. ``asyncio.Lock`` binds to the loop
|
||||
# it is used on; the lock is therefore (re)created whenever the running loop
|
||||
# changes (the Gateway keeps one persistent loop, while the ``asyncio.run``
|
||||
# fallback path spins up a throwaway loop per call — already serialized).
|
||||
_merge_lock: asyncio.Lock | None = None
|
||||
_merge_lock_loop: object = None
|
||||
|
||||
|
||||
def _skill_merge_lock() -> asyncio.Lock:
|
||||
global _merge_lock, _merge_lock_loop
|
||||
loop = asyncio.get_running_loop()
|
||||
if _merge_lock is None or _merge_lock_loop is not loop:
|
||||
_merge_lock = asyncio.Lock()
|
||||
_merge_lock_loop = loop
|
||||
return _merge_lock
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _merge_error_messages(existing: str | None, new: str | None) -> str | None:
|
||||
"""Append a failed attempt's message to the accumulated error text.
|
||||
|
||||
All failures are kept ("都统计下来"), separated and capped at
|
||||
:data:`_MERGED_ERROR_LIMIT` characters.
|
||||
"""
|
||||
new = (new or "").strip()
|
||||
if not new:
|
||||
return existing
|
||||
existing = (existing or "").strip()
|
||||
if not existing:
|
||||
return new
|
||||
if new in existing:
|
||||
# Identical retry failure — no point storing the same line twice.
|
||||
return existing
|
||||
combined = existing + _ERROR_SEP + new
|
||||
if len(combined) > _MERGED_ERROR_LIMIT:
|
||||
combined = combined[: _MERGED_ERROR_LIMIT - 3] + "..."
|
||||
return combined
|
||||
|
||||
|
||||
def _merge_skill_row(existing: ToolCallMetricRow, metric: ToolMetricRecord) -> None:
|
||||
"""Fold another same-run, same-skill call into an existing metric row.
|
||||
|
||||
Outcome rules:
|
||||
- any success in the run -> the row becomes a single ``success`` and earlier
|
||||
failures are dropped ("先报错、最后成功 -> 记一次成功");
|
||||
- all calls failed -> a single ``error`` row whose ``error_message``
|
||||
accumulates every attempt's message ("一直报错 -> 记一次,报错信息都统计下来").
|
||||
"""
|
||||
existing.duration_ms = (existing.duration_ms or 0) + (metric.duration_ms or 0)
|
||||
if metric.status == "success":
|
||||
if existing.status != "success":
|
||||
existing.status = "success"
|
||||
existing.error_category = None
|
||||
existing.error_message = None
|
||||
return
|
||||
# metric.status == "error"
|
||||
if existing.status == "success":
|
||||
# The skill already succeeded once this run — keep that outcome.
|
||||
return
|
||||
existing.status = "error"
|
||||
if metric.error_category:
|
||||
existing.error_category = metric.error_category
|
||||
existing.error_message = _merge_error_messages(existing.error_message, metric.error_message)
|
||||
|
||||
|
||||
def _row_from_metric(metric: ToolMetricRecord) -> ToolCallMetricRow:
|
||||
return ToolCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
tool_name=metric.tool_name,
|
||||
skill_name=metric.skill_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
status=metric.status,
|
||||
error_category=metric.error_category,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
|
||||
|
||||
class SqlToolMetricsStore(ToolMetricsStore):
|
||||
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
# Skill calls are merged per run: the model may touch one skill across
|
||||
# several tool calls / retries, and the admin wants a single row per
|
||||
# (run, skill) instead of one per call. Non-skill tool calls, and skill
|
||||
# calls with no run_id, fall back to a plain insert.
|
||||
if metric.skill_name and metric.run_id:
|
||||
await self._record_skill_merged(metric)
|
||||
return
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(_row_from_metric(metric))
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort; never surface a write failure to the runtime.
|
||||
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
async def _record_skill_merged(self, metric: ToolMetricRecord) -> None:
|
||||
"""Upsert a per-(run, skill) row, folding repeated calls into one."""
|
||||
try:
|
||||
async with _skill_merge_lock():
|
||||
async with self._sf() as session:
|
||||
existing = await session.scalar(
|
||||
select(ToolCallMetricRow)
|
||||
.where(
|
||||
ToolCallMetricRow.run_id == metric.run_id,
|
||||
ToolCallMetricRow.skill_name == metric.skill_name,
|
||||
)
|
||||
.order_by(ToolCallMetricRow.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if existing is None:
|
||||
session.add(_row_from_metric(metric))
|
||||
else:
|
||||
_merge_skill_row(existing, metric)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to merge tool_call_metrics for skill %s (run %s)",
|
||||
metric.skill_name,
|
||||
metric.run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _apply_filters(self, stmt, query: ToolMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
|
||||
if query.tool_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
|
||||
if query.skill_only:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
|
||||
if query.skill_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(ToolCallMetricRow.status == query.status)
|
||||
if query.error_category:
|
||||
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row.to_dict() for row in result.scalars()]
|
||||
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page = await self.list(
|
||||
ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
tool_name=query.tool_name,
|
||||
skill_only=query.skill_only,
|
||||
skill_name=query.skill_name,
|
||||
status=query.status,
|
||||
error_category=query.error_category,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
# Count success/error per skill. Force the skill-only filter so rows
|
||||
# without a skill_name (plain tool calls) never leak into the result.
|
||||
scoped = ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
skill_only=True,
|
||||
skill_name=query.skill_name,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
)
|
||||
stmt = select(
|
||||
ToolCallMetricRow.skill_name,
|
||||
ToolCallMetricRow.status,
|
||||
func.count().label("n"),
|
||||
)
|
||||
stmt = self._apply_filters(stmt, scoped).group_by(
|
||||
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
|
||||
)
|
||||
agg: dict[str, dict[str, int]] = {}
|
||||
async with self._sf() as session:
|
||||
for skill_name, status, n in (await session.execute(stmt)).all():
|
||||
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
|
||||
bucket["total"] += n
|
||||
bucket["success" if status == "success" else "error"] += n
|
||||
return [
|
||||
{"skill_name": name, **counts}
|
||||
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
|
||||
]
|
||||
Reference in new issue
Block a user