大模型速度增加提示
This commit is contained in:
1 parent
4e764350a4
commit
3ea33e1dee
52 files changed
+4673
-176
No files matched your search
+335
@@ -0,0 +1,335 @@
|
||||
"""Middleware that records per-LLM-call metrics for the admin audit page.
|
||||
|
||||
Captures **every** provider invocation regardless of the frontend speed-meter
|
||||
toggle: the goal is a complete admin-side audit log of LLM usage, including
|
||||
failures. The user-facing speed indicator (controlled by
|
||||
``LocalSettings.display.streamSpeedMeterEnabled``) is a separate, purely
|
||||
cosmetic feature — turning it off does not stop this recording.
|
||||
|
||||
Each call ends with one row in ``llm_call_metrics``:
|
||||
- Successes record duration, token counts, and ``tokens_per_sec``.
|
||||
- Errors record the duration spent before the failure, the exception class as
|
||||
``error_type`` and a one-line ``error_message``.
|
||||
|
||||
Writes go through a best-effort store: a DB outage logs a warning and the
|
||||
agent runtime keeps streaming. We intentionally place this middleware
|
||||
*inside* :class:`LLMErrorHandlingMiddleware` (i.e. nested deeper in the
|
||||
``wrap_model_call`` chain) so we observe the **real** provider call timings —
|
||||
not the retry-wrapped successful response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, override
|
||||
|
||||
from langchain.agents import AgentState
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain.agents.middleware.types import (
|
||||
ModelCallResult,
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
)
|
||||
from langchain_core.messages import AIMessage
|
||||
from langgraph.errors import GraphBubbleUp
|
||||
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
from deerflow.persistence.llm_metrics import LlmMetricRecord, make_llm_metrics_store
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricsStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _safe_get_thread_id(request: ModelRequest) -> str | None:
|
||||
state = getattr(request, "state", None)
|
||||
if isinstance(state, dict):
|
||||
thread_id = state.get("thread_id")
|
||||
if isinstance(thread_id, str) and thread_id:
|
||||
return thread_id
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
config = getattr(runtime, "config", None)
|
||||
configurable = (config or {}).get("configurable", {}) if isinstance(config, dict) else {}
|
||||
thread_id = configurable.get("thread_id") if isinstance(configurable, dict) else None
|
||||
if isinstance(thread_id, str) and thread_id:
|
||||
return thread_id
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_run_id(request: ModelRequest) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
config = getattr(runtime, "config", None)
|
||||
if isinstance(config, dict):
|
||||
metadata = config.get("metadata") if isinstance(config.get("metadata"), dict) else {}
|
||||
if isinstance(metadata, dict):
|
||||
for key in ("run_id", "langgraph_run_id"):
|
||||
value = metadata.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_model_name(request: ModelRequest) -> str | None:
|
||||
model = getattr(request, "model", None)
|
||||
for attr in ("model_name", "model", "name"):
|
||||
value = getattr(model, attr, None)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _safe_get_agent_name(request: ModelRequest) -> str | None:
|
||||
runtime = getattr(request, "runtime", None)
|
||||
if runtime is None:
|
||||
return None
|
||||
context = getattr(runtime, "context", None)
|
||||
if isinstance(context, dict):
|
||||
for key in ("agent_name", "agent_id"):
|
||||
value = context.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _extract_usage(response: Any) -> dict[str, int]:
|
||||
"""Pull token counts out of an AIMessage, falling back to provider-native shapes.
|
||||
|
||||
The handler in ``wrap_model_call`` actually hands us a
|
||||
:class:`langchain.agents.middleware.types.ModelResponse` dataclass whose
|
||||
``.result`` is a ``list[BaseMessage]``. Earlier versions of this helper
|
||||
only matched ``AIMessage`` and ``list``, which meant we silently dropped
|
||||
every successful call and stored None for input/output/total. To stay
|
||||
robust we now also handle ``ModelResponse`` (and ``ExtendedModelResponse``,
|
||||
which wraps a ModelResponse), and we walk the messages list for the first
|
||||
``AIMessage``.
|
||||
|
||||
LangChain normalises usage into ``message.usage_metadata`` with the keys
|
||||
``input_tokens`` / ``output_tokens`` / ``total_tokens``. Several
|
||||
OpenAI-compatible providers (DeepSeek, vLLM, some Azure deployments) leave
|
||||
that dict partial — typically only ``total_tokens`` is populated — and the
|
||||
real breakdown sits under ``response_metadata.token_usage`` with the
|
||||
provider-native key names ``prompt_tokens`` / ``completion_tokens``. We
|
||||
merge both so the admin audit view always has the most complete numbers
|
||||
available.
|
||||
"""
|
||||
message: AIMessage | None = None
|
||||
|
||||
def _find_ai(candidate: Any) -> AIMessage | None:
|
||||
if isinstance(candidate, AIMessage):
|
||||
return candidate
|
||||
if isinstance(candidate, list):
|
||||
for item in candidate:
|
||||
found = _find_ai(item)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
message = _find_ai(response)
|
||||
if message is None:
|
||||
# ModelResponse / ExtendedModelResponse — defined as dataclasses with a
|
||||
# ``result`` field holding ``list[BaseMessage]``. Recurse via attribute
|
||||
# access so we don't tightly couple to the dataclass class itself.
|
||||
for attr in ("result", "model_response"):
|
||||
inner = getattr(response, attr, None)
|
||||
if inner is None:
|
||||
continue
|
||||
message = _find_ai(inner)
|
||||
if message is None and attr == "model_response":
|
||||
inner_result = getattr(inner, "result", None)
|
||||
message = _find_ai(inner_result)
|
||||
if message is not None:
|
||||
break
|
||||
if message is None:
|
||||
return {}
|
||||
|
||||
out: dict[str, int] = {}
|
||||
|
||||
usage = getattr(message, "usage_metadata", None) or {}
|
||||
if isinstance(usage, dict):
|
||||
for key in ("input_tokens", "output_tokens", "total_tokens"):
|
||||
coerced = _coerce_int(usage.get(key))
|
||||
if coerced is not None:
|
||||
out[key] = coerced
|
||||
|
||||
response_metadata = getattr(message, "response_metadata", None) or {}
|
||||
token_usage = response_metadata.get("token_usage") if isinstance(response_metadata, dict) else None
|
||||
if isinstance(token_usage, dict):
|
||||
# Map both the LangChain-canonical names and OpenAI-native names; later
|
||||
# writes win so a more-precise provider-native value can override a
|
||||
# zero placeholder from usage_metadata.
|
||||
candidates = {
|
||||
"input_tokens": ("input_tokens", "prompt_tokens"),
|
||||
"output_tokens": ("output_tokens", "completion_tokens"),
|
||||
"total_tokens": ("total_tokens",),
|
||||
}
|
||||
for canonical, keys in candidates.items():
|
||||
for key in keys:
|
||||
coerced = _coerce_int(token_usage.get(key))
|
||||
if coerced is not None and coerced > 0:
|
||||
out[canonical] = coerced
|
||||
break
|
||||
|
||||
# Derive a missing or implausibly-zero field if the other two are present.
|
||||
# Some providers report ``input_tokens=0`` / ``output_tokens=0`` next to a
|
||||
# real ``total_tokens`` — treat those zeros as missing so we don't silently
|
||||
# store nonsense breakdowns.
|
||||
def _missing_or_zero(key: str) -> bool:
|
||||
return out.get(key) in (None, 0)
|
||||
|
||||
if "total_tokens" not in out and out.get("input_tokens") and out.get("output_tokens"):
|
||||
out["total_tokens"] = out["input_tokens"] + out["output_tokens"]
|
||||
if _missing_or_zero("output_tokens") and out.get("total_tokens") and out.get("input_tokens"):
|
||||
derived = out["total_tokens"] - out["input_tokens"]
|
||||
if derived > 0:
|
||||
out["output_tokens"] = derived
|
||||
if _missing_or_zero("input_tokens") and out.get("total_tokens") and out.get("output_tokens"):
|
||||
derived = out["total_tokens"] - out["output_tokens"]
|
||||
if derived > 0:
|
||||
out["input_tokens"] = derived
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _classify_exception(exc: BaseException) -> tuple[str, str]:
|
||||
"""Return (error_type, error_message). Message is trimmed to 500 chars."""
|
||||
name = exc.__class__.__name__
|
||||
raw = (str(exc) or name).strip()
|
||||
return name, (raw[:500] if len(raw) > 500 else raw)
|
||||
|
||||
|
||||
class LlmMetricsMiddleware(AgentMiddleware[AgentState]):
|
||||
"""Wrap each model call to time it and persist a metric row."""
|
||||
|
||||
def __init__(self, *, store: LlmMetricsStore | None = None, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._store_override = store
|
||||
|
||||
def _store(self) -> LlmMetricsStore | None:
|
||||
if self._store_override is not None:
|
||||
return self._store_override
|
||||
try:
|
||||
sf = get_session_factory()
|
||||
except Exception:
|
||||
return None
|
||||
return make_llm_metrics_store(sf)
|
||||
|
||||
def _build_base_record(self, request: ModelRequest) -> LlmMetricRecord:
|
||||
return LlmMetricRecord(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=_now(),
|
||||
user_id=get_effective_user_id(),
|
||||
thread_id=_safe_get_thread_id(request),
|
||||
run_id=_safe_get_run_id(request),
|
||||
agent_name=_safe_get_agent_name(request),
|
||||
model_name=_safe_get_model_name(request),
|
||||
)
|
||||
|
||||
def _finalize_success(self, record: LlmMetricRecord, response: ModelResponse, started: float) -> None:
|
||||
elapsed = max(0.0, time.monotonic() - started)
|
||||
record.duration_ms = int(elapsed * 1000)
|
||||
usage = _extract_usage(response)
|
||||
record.input_tokens = usage.get("input_tokens")
|
||||
record.output_tokens = usage.get("output_tokens")
|
||||
record.total_tokens = usage.get("total_tokens") or (
|
||||
(record.input_tokens or 0) + (record.output_tokens or 0) or None
|
||||
)
|
||||
if record.output_tokens is not None and elapsed > 0:
|
||||
record.tokens_per_sec = round(record.output_tokens / elapsed, 3)
|
||||
record.status = "success"
|
||||
|
||||
def _finalize_error(self, record: LlmMetricRecord, exc: BaseException, started: float) -> None:
|
||||
record.duration_ms = int(max(0.0, time.monotonic() - started) * 1000)
|
||||
record.status = "error"
|
||||
record.error_type, record.error_message = _classify_exception(exc)
|
||||
|
||||
def _flush(self, record: LlmMetricRecord) -> None:
|
||||
store = self._store()
|
||||
if store is None:
|
||||
return
|
||||
try:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None and loop.is_running():
|
||||
# Fire-and-forget — never let metric IO block the request path.
|
||||
loop.create_task(store.record(record))
|
||||
else:
|
||||
asyncio.run(store.record(record))
|
||||
except Exception:
|
||||
logger.warning("LlmMetricsMiddleware: failed to flush metric %s", record.id, exc_info=True)
|
||||
|
||||
@override
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelCallResult:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = handler(request)
|
||||
except GraphBubbleUp:
|
||||
# LangGraph control flow — not a real LLM failure.
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_success(record, response, started)
|
||||
self._flush(record)
|
||||
return response
|
||||
|
||||
@override
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelCallResult:
|
||||
record = self._build_base_record(request)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = await handler(request)
|
||||
except GraphBubbleUp:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._finalize_error(record, exc, started)
|
||||
self._flush(record)
|
||||
raise
|
||||
self._finalize_success(record, response, started)
|
||||
self._flush(record)
|
||||
return response
|
||||
+8
@@ -96,6 +96,14 @@ def _build_runtime_middlewares(
|
||||
|
||||
middlewares.append(LLMErrorHandlingMiddleware(app_config=app_config))
|
||||
|
||||
# LLM call metrics — always on, even when the user-facing speed indicator
|
||||
# is disabled. We append *after* the retry middleware so the metrics
|
||||
# middleware sits deeper in the wrap chain and observes individual provider
|
||||
# attempts (including the failed ones).
|
||||
from deerflow.agents.middlewares.llm_metrics_middleware import LlmMetricsMiddleware
|
||||
|
||||
middlewares.append(LlmMetricsMiddleware())
|
||||
|
||||
# Guardrail middleware (if configured)
|
||||
guardrails_config = app_config.guardrails
|
||||
if guardrails_config.enabled and guardrails_config.provider:
|
||||
|
||||
+6
-1
@@ -45,7 +45,12 @@ class McpServerConfig(BaseModel):
|
||||
headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers to send (for sse or http type)")
|
||||
oauth: McpOAuthConfig | None = Field(default=None, description="OAuth configuration (for sse or http type)")
|
||||
description: str = Field(default="", description="Human-readable description of what this MCP server provides")
|
||||
model_config = ConfigDict(extra="allow")
|
||||
disabled_tools: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="List of tool names (without server-name prefix) to hide from the agent",
|
||||
alias="disabledTools",
|
||||
)
|
||||
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
||||
|
||||
|
||||
class SkillStateConfig(BaseModel):
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""System-wide runtime settings editable from the admin UI.
|
||||
|
||||
Backed by a single JSON file under the DeerFlow runtime home (default
|
||||
``.deer-flow/system_settings.json``). Unlike :mod:`extensions_config` this
|
||||
file is **not** part of the project source tree — it is per-deployment
|
||||
state, and it stays writable at runtime.
|
||||
|
||||
The only feature stored here at the moment is the **prompt prefix toggle**
|
||||
shown in the chat input: an administrator can configure a label and a
|
||||
prefix string, and any user can flip the toggle on/off; when enabled, the
|
||||
prefix is prepended to the first human message of a new thread.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SETTINGS_FILE = "system_settings.json"
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
class PromptPrefixSettings(BaseModel):
|
||||
"""Admin-configurable first-message prompt prefix."""
|
||||
|
||||
enabled: bool = Field(default=False, description="Whether the prefix toggle is offered to users at all")
|
||||
switch_label: str = Field(default="加入立场前缀", description="Toggle label shown in the chat input")
|
||||
prompt_prefix: str = Field(default="", description="Text prepended to the first user message when enabled")
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
class SystemSettings(BaseModel):
|
||||
"""Top-level system settings document."""
|
||||
|
||||
prompt_prefix: PromptPrefixSettings = Field(default_factory=PromptPrefixSettings)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
def _settings_path() -> Path:
|
||||
home = runtime_home()
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
return home / _SETTINGS_FILE
|
||||
|
||||
|
||||
def load_system_settings() -> SystemSettings:
|
||||
"""Read the on-disk system settings, returning defaults when missing."""
|
||||
path = _settings_path()
|
||||
if not path.is_file():
|
||||
return SystemSettings()
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
logger.warning("System settings file %s is not a JSON object; ignoring", path)
|
||||
return SystemSettings()
|
||||
return SystemSettings.model_validate(raw)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read system settings from %s: %s", path, exc)
|
||||
return SystemSettings()
|
||||
|
||||
|
||||
def save_system_settings(settings: SystemSettings) -> SystemSettings:
|
||||
"""Persist the settings document atomically, returning the saved value."""
|
||||
path = _settings_path()
|
||||
payload = settings.model_dump(mode="json")
|
||||
with _lock:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
return settings
|
||||
@@ -121,6 +121,25 @@ async def get_mcp_tools() -> list[BaseTool]:
|
||||
|
||||
# Get all tools from all servers
|
||||
tools = await client.get_tools()
|
||||
|
||||
# Filter out tools listed in each server's `disabled_tools`. Tool names from
|
||||
# MultiServerMCPClient are prefixed with the server name (because we pass
|
||||
# `tool_name_prefix=True`), so we compare against `<server>_<tool>` here.
|
||||
disabled_full_names: set[str] = set()
|
||||
for server_name, server_cfg in extensions_config.get_enabled_mcp_servers().items():
|
||||
for tool_name in server_cfg.disabled_tools or []:
|
||||
name = (tool_name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
disabled_full_names.add(f"{server_name}_{name}")
|
||||
disabled_full_names.add(name)
|
||||
if disabled_full_names:
|
||||
kept = [t for t in tools if t.name not in disabled_full_names]
|
||||
removed = len(tools) - len(kept)
|
||||
if removed:
|
||||
logger.info(f"Filtered out {removed} disabled MCP tool(s)")
|
||||
tools = kept
|
||||
|
||||
logger.info(f"Successfully loaded {len(tools)} tool(s) from MCP servers")
|
||||
|
||||
# Patch tools to support sync invocation, as deerflow client streams synchronously
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence layer for per-LLM-call metrics.
|
||||
|
||||
Each completed LLM invocation (success or failure) writes a single row so
|
||||
administrators can audit usage and speed. Designed to be cheap to write —
|
||||
the middleware fires once per provider call and never blocks streaming.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"LlmMetricRecord",
|
||||
"LlmMetricsStore",
|
||||
"SqlLlmMetricsStore",
|
||||
"make_llm_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlLlmMetricsStore(session_factory)
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Abstract repository contract for LLM call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricRecord:
|
||||
"""In-memory shape of a metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
model_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
tokens_per_sec: float | None = None
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
model_name: str | None = None
|
||||
status: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class LlmMetricsStore(ABC):
|
||||
"""Persistence contract for the metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""ORM model for per-LLM-call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
"""One row per completed (or failed) LLM provider invocation."""
|
||||
|
||||
__tablename__ = "llm_call_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
# Wall-clock duration of the provider invocation in milliseconds. Always set.
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
input_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
output_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
total_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
# Output tokens / seconds; null when there is no completion (errors).
|
||||
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
# "success" or "error". Errors store the classification and original message
|
||||
# so admins can scan for transient vs auth/quota issues.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"model_name": self.model_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
"tokens_per_sec": self.tokens_per_sec,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""SQLAlchemy-backed LLM metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
row = LlmCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
model_name=metric.model_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
input_tokens=metric.input_tokens,
|
||||
output_tokens=metric.output_tokens,
|
||||
total_tokens=metric.total_tokens,
|
||||
tokens_per_sec=metric.tokens_per_sec,
|
||||
status=metric.status,
|
||||
error_type=metric.error_type,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort. Never let a persistence failure surface
|
||||
# to the agent runtime — log and move on.
|
||||
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
def _apply_filters(self, stmt, query: LlmMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(LlmCallMetricRow.status == query.status)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
# Page through the result so the Excel export stays bounded in memory
|
||||
# for large date windows.
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page_query = LlmMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
model_name=query.model_name,
|
||||
status=query.status,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
page = await self.list(page_query)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""Create llm_call_metrics table.
|
||||
|
||||
Revision ID: 20260514_02
|
||||
Revises: 20260514_01
|
||||
Create Date: 2026-05-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260514_02"
|
||||
down_revision = "20260514_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"llm_call_metrics",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=True),
|
||||
sa.Column("run_id", sa.String(64), nullable=True),
|
||||
sa.Column("agent_name", sa.String(128), nullable=True),
|
||||
sa.Column("model_name", sa.String(128), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("input_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("output_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("total_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("tokens_per_sec", sa.Float(), nullable=True),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
|
||||
sa.Column("error_type", sa.String(64), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_llm_call_metrics_created_at", "llm_call_metrics", ["created_at"])
|
||||
op.create_index("ix_llm_call_metrics_user_id", "llm_call_metrics", ["user_id"])
|
||||
op.create_index("ix_llm_call_metrics_model_name", "llm_call_metrics", ["model_name"])
|
||||
op.create_index("ix_llm_call_metrics_status", "llm_call_metrics", ["status"])
|
||||
op.create_index("ix_llm_call_metrics_thread_id", "llm_call_metrics", ["thread_id"])
|
||||
op.create_index("ix_llm_call_metrics_run_id", "llm_call_metrics", ["run_id"])
|
||||
op.create_index("ix_llm_call_metrics_agent_name", "llm_call_metrics", ["agent_name"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
op.drop_table("llm_call_metrics")
|
||||
+4
@@ -17,7 +17,9 @@ there is no matching entity directory.
|
||||
|
||||
from deerflow.persistence.agents.model import AgentRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
ScheduledTaskDeliveryProfileRow,
|
||||
@@ -32,6 +34,8 @@ from deerflow.persistence.user.model import UserRow
|
||||
__all__ = [
|
||||
"FeedbackRow",
|
||||
"AgentRow",
|
||||
"LlmCallMetricRow",
|
||||
"RecommendedQuestionRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
|
||||
+10
-2
@@ -247,7 +247,11 @@ class ScheduledTaskService:
|
||||
accessible = await self._accessible_task(user_id, task_id)
|
||||
if accessible is None:
|
||||
return []
|
||||
return await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
|
||||
runs = await self.store.list_runs(task_id, accessible["user_id"], limit=limit)
|
||||
can_edit = accessible.get("role") == "owner"
|
||||
for run in runs:
|
||||
run["can_edit"] = can_edit
|
||||
return runs
|
||||
|
||||
async def get_run(self, user_id: str, run_id: str) -> dict[str, Any] | None:
|
||||
run = await self.store.get_run(run_id, user_id)
|
||||
@@ -316,7 +320,7 @@ class ScheduledTaskService:
|
||||
except ValueError as exc:
|
||||
raise PermissionError("Refusing to edit file outside the run archive") from exc
|
||||
|
||||
archive_path.write_text(new_content, encoding="utf-8")
|
||||
archive_path.write_text(new_content, encoding="utf-8", newline="")
|
||||
new_size = archive_path.stat().st_size
|
||||
|
||||
# Reflect the new size in the run row so subsequent list/preview calls
|
||||
@@ -342,6 +346,10 @@ class ScheduledTaskService:
|
||||
shutil.rmtree(archive_dir)
|
||||
except Exception:
|
||||
logger.warning("Failed to delete scheduled task run archive %s", archive_dir, exc_info=True)
|
||||
return True
|
||||
candidate = await self.store.get_run_any(run_id)
|
||||
if candidate is not None and await self.store.get_subscription(candidate["task_id"], user_id) is not None:
|
||||
raise PermissionError("Only the run owner may delete scheduled task runs")
|
||||
return deleted
|
||||
|
||||
async def get_delivery_profile(self, user_id: str) -> dict[str, Any]:
|
||||
|
||||
Reference in new issue
Block a user