Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xuhd
This commit is contained in:
commit
64492bec43
46 files changed
+3298
-313
No files matched your search
+8
@@ -131,6 +131,14 @@ def _build_runtime_middlewares(
|
||||
|
||||
middlewares.append(SandboxAuditMiddleware())
|
||||
middlewares.append(ToolErrorHandlingMiddleware())
|
||||
|
||||
# Tool / skill call metrics — appended last so it sits *deepest* in the
|
||||
# wrap_tool_call chain: its handler is the raw tool, so it observes the
|
||||
# unconverted exception (before ToolErrorHandlingMiddleware turns it into
|
||||
# an error ToolMessage) and the raw return value.
|
||||
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
|
||||
|
||||
middlewares.append(ToolMetricsMiddleware())
|
||||
return middlewares
|
||||
|
||||
|
||||
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
"""Middleware that records per-tool-call metrics for the admin audit page.
|
||||
|
||||
Every tool / skill invocation ends with one row in ``tool_call_metrics``:
|
||||
- Successes record the wall-clock duration.
|
||||
- Failures record a coarse ``error_category`` plus a one-line message.
|
||||
|
||||
Two failure shapes are captured:
|
||||
|
||||
1. **Raised exceptions** — the tool blew up. We record, then re-raise so the
|
||||
outer :class:`ToolErrorHandlingMiddleware` still converts it into an error
|
||||
``ToolMessage`` for the model.
|
||||
2. **Self-handled errors** — search/retrieval tools (Serper, Tavily, Jina, …)
|
||||
catch their own HTTP 429/403/timeout and *return* an error payload instead
|
||||
of raising. Those never reach an ``except`` block, so we also inspect the
|
||||
returned ``ToolMessage``: an ``status == "error"`` message, or content that
|
||||
looks like an error string, is bucketed too.
|
||||
|
||||
This middleware is appended **after** ``ToolErrorHandlingMiddleware`` so it
|
||||
sits deepest in the ``wrap_tool_call`` chain — its handler is the raw tool, so
|
||||
it observes the unconverted exception and the raw return value.
|
||||
|
||||
Writes go through a best-effort store: a DB outage logs a warning and the
|
||||
agent runtime keeps running.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langgraph.errors import GraphBubbleUp
|
||||
from langgraph.prebuilt.tool_node import ToolCallRequest
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.tool_metrics import ToolMetricRecord, make_tool_metrics_store
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _trim(text: str, limit: int = 500) -> str:
|
||||
text = (text or "").strip()
|
||||
return text if len(text) <= limit else text[: limit - 3] + "..."
|
||||
|
||||
|
||||
def _classify_error(text: str) -> str:
|
||||
"""Bucket an error string into a coarse, admin-friendly category."""
|
||||
t = (text or "").lower()
|
||||
if any(k in t for k in ("429", "rate limit", "ratelimit", "too many request", "quota")):
|
||||
return "rate_limited"
|
||||
if any(k in t for k in ("403", "forbidden", "ip ban", "ip block", "blocked", "access denied", "banned")):
|
||||
return "ip_blocked"
|
||||
if any(k in t for k in ("401", "unauthorized", "api key", "apikey", "invalid key", "authentication", "credential")):
|
||||
return "auth_error"
|
||||
if any(k in t for k in ("timeout", "timed out", "deadline")):
|
||||
return "timeout"
|
||||
if any(k in t for k in ("connecterror", "connection error", "connection refused", "network", "dns", "unreachable", "ssl", "proxy")):
|
||||
return "network_error"
|
||||
if any(k in t for k in ("404", "not found")):
|
||||
return "not_found"
|
||||
if any(k in t for k in ("no results", "no result", "0 results", "empty result")):
|
||||
return "empty_result"
|
||||
return "tool_error"
|
||||
|
||||
|
||||
def _content_text(content: Any) -> str:
|
||||
"""Flatten a tool result's content (string or block list) into text."""
|
||||
if content is None:
|
||||
return ""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
text = block.get("text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
return " ".join(parts)
|
||||
return str(content)
|
||||
|
||||
|
||||
def _looks_like_error(text: str) -> bool:
|
||||
"""Heuristically detect a self-handled error payload returned as content.
|
||||
|
||||
Deliberately conservative — only an explicit ``Error:`` prefix or a JSON
|
||||
object whose first key is ``error`` counts, so a normal result that merely
|
||||
mentions the word "error" is not misclassified.
|
||||
"""
|
||||
head = (text or "").lstrip()[:64].lower()
|
||||
return head.startswith("error:") or head.startswith('{"error"') or head.startswith("{'error'")
|
||||
|
||||
|
||||
# Skill tools whose ``name`` argument names one specific Agent Skill directly.
|
||||
_SKILL_NAME_ARG_TOOLS = {"skill_view", "skill_manage", "skill_archive"}
|
||||
|
||||
# Matches a skill directory embedded in any tool-argument path. Skills live
|
||||
# under ``skills/{public,custom}/{name}/`` — both inside the sandbox
|
||||
# (``/mnt/skills/...``) and on the host. The lead agent's system prompt tells
|
||||
# it to load a skill by calling ``read_file`` on that SKILL.md path (it does
|
||||
# NOT call a dedicated "use skill" tool), so the skill name has to be recovered
|
||||
# from the path argument of whatever tool touched it (read_file / bash / ls /
|
||||
# str_replace / write_file ...).
|
||||
_SKILL_PATH_RE = re.compile(r"[\\/]skills[\\/](?:public|custom)[\\/]([^\\/]+)(?:[\\/]|$)")
|
||||
|
||||
|
||||
def _skill_name_from_call(tool_name: str | None, tool_call: Any) -> str | None:
|
||||
"""Recover the Agent Skill a tool call targets, or ``None``.
|
||||
|
||||
Two detection paths:
|
||||
|
||||
1. ``skill_view`` / ``skill_manage`` / ``skill_archive`` take an explicit
|
||||
``name`` argument.
|
||||
2. Every other tool references a skill by *path*. The canonical skill
|
||||
invocation is ``read_file('/mnt/skills/<cat>/<name>/SKILL.md')``; helper
|
||||
files and ``bash`` script runs hit the same directory. So we scan the
|
||||
string arguments for the ``skills/{public,custom}/{name}/`` shape.
|
||||
"""
|
||||
if not isinstance(tool_call, dict):
|
||||
return None
|
||||
args = tool_call.get("args")
|
||||
if not isinstance(args, dict):
|
||||
return None
|
||||
|
||||
# Case 1 — explicit ``name`` argument on the skill tools.
|
||||
if tool_name in _SKILL_NAME_ARG_TOOLS:
|
||||
raw = args.get("name")
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
return raw.strip()
|
||||
|
||||
# Case 2 — a skill path inside any string argument.
|
||||
for value in args.values():
|
||||
if isinstance(value, str):
|
||||
match = _SKILL_PATH_RE.search(value)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def _evaluate_skill_view(text: str) -> tuple[str, str | None, str | None]:
|
||||
"""Classify a ``skill_view`` return.
|
||||
|
||||
``skill_view`` never raises — on failure it returns a plain string
|
||||
(``Skill 'x' not found.`` / ``Error reading skill 'x': ...``); on success
|
||||
it returns the SKILL.md content. So success/failure must be read off the
|
||||
returned text, not an exception or ``status`` flag.
|
||||
"""
|
||||
head = (text or "").lstrip()
|
||||
low = head[:200].lower()
|
||||
if head.startswith("Skill '") and "not found" in low:
|
||||
return "error", "not_found", _trim(text)
|
||||
if low.startswith("error reading skill"):
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
return "success", None, None
|
||||
|
||||
|
||||
def _evaluate_result(result: Any, tool_name: str | None) -> tuple[str, str | None, str | None]:
|
||||
"""Return ``(status, error_category, error_message)`` for a tool return."""
|
||||
if isinstance(result, ToolMessage):
|
||||
text = _content_text(result.content)
|
||||
if getattr(result, "status", None) == "error":
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
# skill_view swallows its own failures into a plain string return.
|
||||
if tool_name == "skill_view":
|
||||
return _evaluate_skill_view(text)
|
||||
if _looks_like_error(text):
|
||||
return "error", _classify_error(text), _trim(text)
|
||||
return "success", None, None
|
||||
# Command (control flow) or any other shape — treat as a successful call.
|
||||
return "success", None, None
|
||||
|
||||
|
||||
class ToolMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Wrap each tool call to time it and persist a metric row."""
|
||||
|
||||
def __init__(self, *, store: ToolMetricsStore | None = None, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._store_override = store
|
||||
|
||||
# -- store / context ----------------------------------------------------
|
||||
|
||||
def _store(self) -> ToolMetricsStore | None:
|
||||
if self._store_override is not None:
|
||||
return self._store_override
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
except Exception:
|
||||
return None
|
||||
return make_tool_metrics_store(sf)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_lookup(request: ToolCallRequest, keys: tuple[str, ...]) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in keys:
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
config = getattr(runtime, "config", None)
|
||||
if isinstance(config, dict):
|
||||
configurable = config.get("configurable") if isinstance(config.get("configurable"), dict) else {}
|
||||
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
|
||||
for bucket in (configurable, metadata):
|
||||
for key in keys:
|
||||
value = bucket.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
def _build_base_record(self, request: ToolCallRequest) -> ToolMetricRecord:
|
||||
tool_call = getattr(request, "tool_call", None) or {}
|
||||
tool_name = tool_call.get("name") if isinstance(tool_call, dict) else None
|
||||
tool_name = str(tool_name) if tool_name else None
|
||||
return ToolMetricRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=_now(),
|
||||
user_id=get_effective_user_id(),
|
||||
thread_id=self._runtime_lookup(request, ("thread_id",)),
|
||||
run_id=self._runtime_lookup(request, ("run_id", "langgraph_run_id")),
|
||||
agent_name=self._runtime_lookup(request, ("agent_name", "agent_id")),
|
||||
skill_name=_skill_name_from_call(tool_name, tool_call),
|
||||
tool_name=tool_name,
|
||||
)
|
||||
|
||||
# -- finalize -----------------------------------------------------------
|
||||
|
||||
def _finalize_result(self, record: ToolMetricRecord, result: Any, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status, record.error_category, record.error_message = _evaluate_result(
|
||||
result, record.tool_name
|
||||
)
|
||||
|
||||
def _finalize_error(self, record: ToolMetricRecord, exc: BaseException, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status = "error"
|
||||
name = exc.__class__.__name__
|
||||
combined = f"{name}: {(str(exc) or name).strip()}"
|
||||
record.error_category = _classify_error(combined)
|
||||
record.error_message = _trim(combined)
|
||||
|
||||
def _flush(self, record: ToolMetricRecord) -> None:
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the tool path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("ToolMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
# -- wrap ---------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def wrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], ToolMessage | Command],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = handler(request)
|
||||
except GraphBubbleUp:
|
||||
# LangGraph control flow — not a tool failure.
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_result(record, result, started)
|
||||
self._flush(record)
|
||||
return result
|
||||
|
||||
@override
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command]],
|
||||
) -> ToolMessage | Command:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = await handler(request)
|
||||
except GraphBubbleUp:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_result(record, result, started)
|
||||
self._flush(record)
|
||||
return result
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
"""Create tool_call_metrics table.
|
||||
|
||||
Revision ID: 20260518_07
|
||||
Revises: 20260518_06
|
||||
Create Date: 2026-05-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260518_07"
|
||||
down_revision = "20260518_06"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("tool_call_metrics"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"tool_call_metrics",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=True),
|
||||
sa.Column("run_id", sa.String(64), nullable=True),
|
||||
sa.Column("agent_name", sa.String(128), nullable=True),
|
||||
sa.Column("tool_name", sa.String(128), nullable=True),
|
||||
sa.Column("skill_name", sa.String(128), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
|
||||
sa.Column("error_category", sa.String(32), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_tool_call_metrics_created_at", "tool_call_metrics", ["created_at"])
|
||||
op.create_index("ix_tool_call_metrics_user_id", "tool_call_metrics", ["user_id"])
|
||||
op.create_index("ix_tool_call_metrics_thread_id", "tool_call_metrics", ["thread_id"])
|
||||
op.create_index("ix_tool_call_metrics_run_id", "tool_call_metrics", ["run_id"])
|
||||
op.create_index("ix_tool_call_metrics_agent_name", "tool_call_metrics", ["agent_name"])
|
||||
op.create_index("ix_tool_call_metrics_tool_name", "tool_call_metrics", ["tool_name"])
|
||||
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
|
||||
op.create_index("ix_tool_call_metrics_status", "tool_call_metrics", ["status"])
|
||||
op.create_index("ix_tool_call_metrics_error_category", "tool_call_metrics", ["error_category"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("tool_call_metrics"):
|
||||
op.drop_table("tool_call_metrics")
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""Add skill_name column to tool_call_metrics.
|
||||
|
||||
``tool_call_metrics`` is created by ``create_all`` at startup. On deployments
|
||||
where the table was materialised *before* the ``skill_name`` column existed,
|
||||
neither ``create_all`` (it never ALTERs) nor revision ``20260518_07`` (it skips
|
||||
when the table already exists) adds the column — leaving MySQL/Postgres with a
|
||||
schema that the ORM model expects a column it does not have. This revision
|
||||
patches the existing table.
|
||||
|
||||
Revision ID: 20260519_01
|
||||
Revises: 20260518_07
|
||||
Create Date: 2026-05-19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260519_01"
|
||||
down_revision = "20260518_07"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def _has_column(table_name: str, column_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(col["name"] == column_name for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name: str, index_name: str) -> bool:
|
||||
if not _has_table(table_name):
|
||||
return False
|
||||
inspector = inspect(op.get_bind())
|
||||
return any(idx["name"] == index_name for idx in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Brand-new DBs: the table is created with the column already present
|
||||
# (by create_all / revision 20260518_07) — nothing to patch.
|
||||
if not _has_table("tool_call_metrics"):
|
||||
return
|
||||
if not _has_column("tool_call_metrics", "skill_name"):
|
||||
op.add_column("tool_call_metrics", sa.Column("skill_name", sa.String(128), nullable=True))
|
||||
if not _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
|
||||
op.create_index("ix_tool_call_metrics_skill_name", "tool_call_metrics", ["skill_name"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_index("tool_call_metrics", "ix_tool_call_metrics_skill_name"):
|
||||
op.drop_index("ix_tool_call_metrics_skill_name", table_name="tool_call_metrics")
|
||||
if _has_column("tool_call_metrics", "skill_name"):
|
||||
op.drop_column("tool_call_metrics", "skill_name")
|
||||
+2
@@ -33,6 +33,7 @@ from deerflow.persistence.skills.model import SkillRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
__all__ = [
|
||||
@@ -53,5 +54,6 @@ __all__ = [
|
||||
"TagRow",
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"UserRow",
|
||||
]
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
"""Persistence layer for per-tool-call (skill-call) metrics.
|
||||
|
||||
Every tool invocation — built-in tools, skills, MCP tools, subagent calls —
|
||||
writes one row recording duration and outcome. Failures are bucketed into a
|
||||
coarse ``error_category`` (rate_limited / ip_blocked / auth_error / timeout /
|
||||
network_error / not_found / empty_result / tool_error) so administrators can
|
||||
spot retrieval outages, blocked IPs, and quota problems at a glance.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.tool_metrics.sql import SqlToolMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"ToolCallMetricRow",
|
||||
"ToolMetricRecord",
|
||||
"ToolMetricsStore",
|
||||
"SqlToolMetricsStore",
|
||||
"make_tool_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_tool_metrics_store(session_factory) -> ToolMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlToolMetricsStore(session_factory)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
"""Abstract repository contract for tool / skill call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricRecord:
|
||||
"""In-memory shape of a tool-call metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
tool_name: str | None = None
|
||||
# The specific Agent Skill this call targets, when the tool is a skill tool
|
||||
# (``skill_view`` / ``skill_manage`` / ``skill_archive``) — taken from the
|
||||
# call's ``name`` argument. Null for every non-skill tool call. This is
|
||||
# what lets the admin see "skill X was invoked, and succeeded/failed".
|
||||
skill_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
status: str = "success"
|
||||
# Coarse failure bucket: rate_limited / ip_blocked / auth_error / timeout /
|
||||
# network_error / not_found / empty_result / tool_error. Null on success.
|
||||
error_category: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
tool_name: str | None = None
|
||||
# When True, keep only rows that target a specific Agent Skill
|
||||
# (``skill_name IS NOT NULL``) — the ``kind=skill`` filter.
|
||||
skill_only: bool = False
|
||||
# Filter to one specific skill by name.
|
||||
skill_name: str | None = None
|
||||
status: str | None = None
|
||||
error_category: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class ToolMetricsStore(ABC):
|
||||
"""Persistence contract for the tool-metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: ToolMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Per-skill aggregate: ``[{skill_name, total, success, error}, ...]``.
|
||||
|
||||
Only rows with a non-null ``skill_name`` are counted, so this answers
|
||||
"which skills were invoked, and how often did each succeed/fail".
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""ORM model for per-tool-call (skill-call) metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ToolCallMetricRow(Base):
|
||||
"""One row per completed (or failed) tool / skill invocation.
|
||||
|
||||
Unlike :class:`LlmCallMetricRow` this also captures *self-handled*
|
||||
failures — search tools that swallow an HTTP 429/403 and return an error
|
||||
payload instead of raising — so an admin can see "retrieval failed" /
|
||||
"IP blocked" events that never surfaced as exceptions.
|
||||
"""
|
||||
|
||||
__tablename__ = "tool_call_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
tool_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# The specific Agent Skill targeted by a skill tool call (from the call's
|
||||
# ``name`` argument); null for non-skill tool calls.
|
||||
skill_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
# "success" or "error".
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
# Coarse failure bucket; null on success.
|
||||
error_category: Mapped[str | None] = mapped_column(String(32), index=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"tool_name": self.tool_name,
|
||||
"skill_name": self.skill_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"status": self.status,
|
||||
"error_category": self.error_category,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
"""SQLAlchemy-backed tool metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery, ToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class SqlToolMetricsStore(ToolMetricsStore):
|
||||
"""SQLite/Postgres/MySQL-backed implementation; writes are fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: ToolMetricRecord) -> None:
|
||||
row = ToolCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
tool_name=metric.tool_name,
|
||||
skill_name=metric.skill_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
status=metric.status,
|
||||
error_category=metric.error_category,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort; never surface a write failure to the runtime.
|
||||
logger.warning("Failed to persist tool_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
def _apply_filters(self, stmt, query: ToolMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(ToolCallMetricRow.user_id == query.user_id)
|
||||
if query.tool_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.tool_name == query.tool_name)
|
||||
if query.skill_only:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name.is_not(None))
|
||||
if query.skill_name:
|
||||
stmt = stmt.where(ToolCallMetricRow.skill_name == query.skill_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(ToolCallMetricRow.status == query.status)
|
||||
if query.error_category:
|
||||
stmt = stmt.where(ToolCallMetricRow.error_category == query.error_category)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(ToolCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(ToolCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [row.to_dict() for row in result.scalars()]
|
||||
|
||||
async def count(self, query: ToolMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(ToolCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: ToolMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page = await self.list(
|
||||
ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
tool_name=query.tool_name,
|
||||
skill_only=query.skill_only,
|
||||
skill_name=query.skill_name,
|
||||
status=query.status,
|
||||
error_category=query.error_category,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
|
||||
async def skill_breakdown(self, query: ToolMetricsQuery) -> list[dict[str, Any]]:
|
||||
# Count success/error per skill. Force the skill-only filter so rows
|
||||
# without a skill_name (plain tool calls) never leak into the result.
|
||||
scoped = ToolMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
skill_only=True,
|
||||
skill_name=query.skill_name,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
)
|
||||
stmt = select(
|
||||
ToolCallMetricRow.skill_name,
|
||||
ToolCallMetricRow.status,
|
||||
func.count().label("n"),
|
||||
)
|
||||
stmt = self._apply_filters(stmt, scoped).group_by(
|
||||
ToolCallMetricRow.skill_name, ToolCallMetricRow.status
|
||||
)
|
||||
agg: dict[str, dict[str, int]] = {}
|
||||
async with self._sf() as session:
|
||||
for skill_name, status, n in (await session.execute(stmt)).all():
|
||||
bucket = agg.setdefault(skill_name, {"total": 0, "success": 0, "error": 0})
|
||||
bucket["total"] += n
|
||||
bucket["success" if status == "success" else "error"] += n
|
||||
return [
|
||||
{"skill_name": name, **counts}
|
||||
for name, counts in sorted(agg.items(), key=lambda kv: kv[1]["total"], reverse=True)
|
||||
]
|
||||
Reference in new issue
Block a user