大模型速度增加提示
This commit is contained in:
1 parent
4e764350a4
commit
3ea33e1dee
52 files changed
+4673
-176
No files matched your search
+25
@@ -0,0 +1,25 @@
|
||||
"""Persistence layer for per-LLM-call metrics.
|
||||
|
||||
Each completed LLM invocation (success or failure) writes a single row so
|
||||
administrators can audit usage and speed. Designed to be cheap to write —
|
||||
the middleware fires once per provider call and never blocks streaming.
|
||||
"""
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llm_metrics.sql import SqlLlmMetricsStore
|
||||
|
||||
__all__ = [
|
||||
"LlmCallMetricRow",
|
||||
"LlmMetricRecord",
|
||||
"LlmMetricsStore",
|
||||
"SqlLlmMetricsStore",
|
||||
"make_llm_metrics_store",
|
||||
]
|
||||
|
||||
|
||||
def make_llm_metrics_store(session_factory) -> LlmMetricsStore | None:
|
||||
"""Return a SQL-backed metrics store, or ``None`` when no DB is configured."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlLlmMetricsStore(session_factory)
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Abstract repository contract for LLM call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricRecord:
|
||||
"""In-memory shape of a metric row before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
user_id: str | None = None
|
||||
thread_id: str | None = None
|
||||
run_id: str | None = None
|
||||
agent_name: str | None = None
|
||||
model_name: str | None = None
|
||||
duration_ms: int = 0
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
tokens_per_sec: float | None = None
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmMetricsQuery:
|
||||
"""Filter set for the admin list / export endpoints."""
|
||||
|
||||
user_id: str | None = None
|
||||
model_name: str | None = None
|
||||
status: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class LlmMetricsStore(ABC):
|
||||
"""Persistence contract for the metrics middleware + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
"""Persist one metric row. Must never raise on best-effort writes."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
"""Return the matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def iter_all(self, query: LlmMetricsQuery):
|
||||
"""Yield rows matching ``query`` without pagination — used by export."""
|
||||
raise NotImplementedError
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
"""ORM model for per-LLM-call metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
class LlmCallMetricRow(Base):
|
||||
"""One row per completed (or failed) LLM provider invocation."""
|
||||
|
||||
__tablename__ = "llm_call_metrics"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Wall-clock time the LLM call ended. Indexed because admin views always
|
||||
# sort/filter by recency.
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True)
|
||||
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
thread_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
|
||||
# Wall-clock duration of the provider invocation in milliseconds. Always set.
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
input_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
output_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
total_tokens: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
# Output tokens / seconds; null when there is no completion (errors).
|
||||
tokens_per_sec: Mapped[float | None] = mapped_column(Float)
|
||||
|
||||
# "success" or "error". Errors store the classification and original message
|
||||
# so admins can scan for transient vs auth/quota issues.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success", index=True)
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if isinstance(self.created_at, datetime) else self.created_at,
|
||||
"user_id": self.user_id,
|
||||
"thread_id": self.thread_id,
|
||||
"run_id": self.run_id,
|
||||
"agent_name": self.agent_name,
|
||||
"model_name": self.model_name,
|
||||
"duration_ms": self.duration_ms,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
"tokens_per_sec": self.tokens_per_sec,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""SQLAlchemy-backed LLM metrics repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery, LlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _row_to_dict(row: LlmCallMetricRow) -> dict[str, Any]:
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def _normalize_created_at(value: datetime) -> datetime:
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
class SqlLlmMetricsStore(LlmMetricsStore):
|
||||
"""SQLite/Postgres-backed implementation; writes are intentionally fire-and-forget."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def record(self, metric: LlmMetricRecord) -> None:
|
||||
row = LlmCallMetricRow(
|
||||
id=metric.id,
|
||||
created_at=_normalize_created_at(metric.created_at),
|
||||
user_id=metric.user_id,
|
||||
thread_id=metric.thread_id,
|
||||
run_id=metric.run_id,
|
||||
agent_name=metric.agent_name,
|
||||
model_name=metric.model_name,
|
||||
duration_ms=metric.duration_ms,
|
||||
input_tokens=metric.input_tokens,
|
||||
output_tokens=metric.output_tokens,
|
||||
total_tokens=metric.total_tokens,
|
||||
tokens_per_sec=metric.tokens_per_sec,
|
||||
status=metric.status,
|
||||
error_type=metric.error_type,
|
||||
error_message=metric.error_message,
|
||||
)
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
# Metrics are best-effort. Never let a persistence failure surface
|
||||
# to the agent runtime — log and move on.
|
||||
logger.warning("Failed to persist llm_call_metrics row %s", metric.id, exc_info=True)
|
||||
|
||||
def _apply_filters(self, stmt, query: LlmMetricsQuery):
|
||||
if query.user_id:
|
||||
stmt = stmt.where(LlmCallMetricRow.user_id == query.user_id)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(LlmCallMetricRow.model_name == query.model_name)
|
||||
if query.status:
|
||||
stmt = stmt.where(LlmCallMetricRow.status == query.status)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at >= _normalize_created_at(query.since))
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(LlmCallMetricRow.created_at <= _normalize_created_at(query.until))
|
||||
return stmt
|
||||
|
||||
async def list(self, query: LlmMetricsQuery) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
stmt = stmt.order_by(desc(LlmCallMetricRow.created_at)).limit(max(1, min(query.limit, 1000))).offset(max(0, query.offset))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def count(self, query: LlmMetricsQuery) -> int:
|
||||
stmt = select(func.count()).select_from(LlmCallMetricRow)
|
||||
stmt = self._apply_filters(stmt, query)
|
||||
async with self._sf() as session:
|
||||
return (await session.scalar(stmt)) or 0
|
||||
|
||||
async def iter_all(self, query: LlmMetricsQuery) -> AsyncIterator[dict[str, Any]]:
|
||||
# Page through the result so the Excel export stays bounded in memory
|
||||
# for large date windows.
|
||||
page_size = 500
|
||||
offset = 0
|
||||
while True:
|
||||
page_query = LlmMetricsQuery(
|
||||
user_id=query.user_id,
|
||||
model_name=query.model_name,
|
||||
status=query.status,
|
||||
since=query.since,
|
||||
until=query.until,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
page = await self.list(page_query)
|
||||
if not page:
|
||||
return
|
||||
for item in page:
|
||||
yield item
|
||||
if len(page) < page_size:
|
||||
return
|
||||
offset += page_size
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
"""Create llm_call_metrics table.
|
||||
|
||||
Revision ID: 20260514_02
|
||||
Revises: 20260514_01
|
||||
Create Date: 2026-05-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
|
||||
revision = "20260514_02"
|
||||
down_revision = "20260514_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
inspector = inspect(op.get_bind())
|
||||
return table_name in inspector.get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"llm_call_metrics",
|
||||
sa.Column("id", sa.String(64), primary_key=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("user_id", sa.String(64), nullable=True),
|
||||
sa.Column("thread_id", sa.String(64), nullable=True),
|
||||
sa.Column("run_id", sa.String(64), nullable=True),
|
||||
sa.Column("agent_name", sa.String(128), nullable=True),
|
||||
sa.Column("model_name", sa.String(128), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("input_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("output_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("total_tokens", sa.Integer(), nullable=True),
|
||||
sa.Column("tokens_per_sec", sa.Float(), nullable=True),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="success"),
|
||||
sa.Column("error_type", sa.String(64), nullable=True),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index("ix_llm_call_metrics_created_at", "llm_call_metrics", ["created_at"])
|
||||
op.create_index("ix_llm_call_metrics_user_id", "llm_call_metrics", ["user_id"])
|
||||
op.create_index("ix_llm_call_metrics_model_name", "llm_call_metrics", ["model_name"])
|
||||
op.create_index("ix_llm_call_metrics_status", "llm_call_metrics", ["status"])
|
||||
op.create_index("ix_llm_call_metrics_thread_id", "llm_call_metrics", ["thread_id"])
|
||||
op.create_index("ix_llm_call_metrics_run_id", "llm_call_metrics", ["run_id"])
|
||||
op.create_index("ix_llm_call_metrics_agent_name", "llm_call_metrics", ["agent_name"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _has_table("llm_call_metrics"):
|
||||
op.drop_table("llm_call_metrics")
|
||||
+4
@@ -17,7 +17,9 @@ there is no matching entity directory.
|
||||
|
||||
from deerflow.persistence.agents.model import AgentRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.recommended_questions.model import RecommendedQuestionRow
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.scheduled_tasks.model import (
|
||||
ScheduledTaskDeliveryProfileRow,
|
||||
@@ -32,6 +34,8 @@ from deerflow.persistence.user.model import UserRow
|
||||
__all__ = [
|
||||
"FeedbackRow",
|
||||
"AgentRow",
|
||||
"LlmCallMetricRow",
|
||||
"RecommendedQuestionRow",
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
|
||||
Reference in new issue
Block a user