sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+104
@@ -0,0 +1,104 @@
|
||||
"""Persistence for model-stream truncation events (模型截断记录).
|
||||
|
||||
The patched OpenAI-compatible streaming layer calls
|
||||
:func:`record_stream_truncation_event` whenever a provider stream ends
|
||||
abnormally. The write is fire-and-forget: it is routed onto the engine's home
|
||||
loop via :func:`fire_and_forget_db_write` and must never disturb the model
|
||||
stream itself. Rows are capped at one month by the SQL repository.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
from deerflow.persistence.stream_truncation.sql import SqlStreamTruncationStore
|
||||
|
||||
__all__ = [
|
||||
"RETENTION_DAYS",
|
||||
"SqlStreamTruncationStore",
|
||||
"StreamTruncationEvent",
|
||||
"StreamTruncationEventRow",
|
||||
"StreamTruncationQuery",
|
||||
"StreamTruncationStore",
|
||||
"make_stream_truncation_store",
|
||||
"record_stream_truncation_event",
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_store_cache: SqlStreamTruncationStore | None = None
|
||||
_store_cache_failed = False
|
||||
|
||||
|
||||
def make_stream_truncation_store(session_factory) -> StreamTruncationStore | None:
|
||||
"""Return a SQL-backed truncation store, or ``None`` without a DB."""
|
||||
if session_factory is None:
|
||||
return None
|
||||
return SqlStreamTruncationStore(session_factory)
|
||||
|
||||
|
||||
def record_stream_truncation_event(
|
||||
*,
|
||||
model_name: str | None,
|
||||
event_type: str,
|
||||
strict_mode: bool = False,
|
||||
finish_reason: str = "none",
|
||||
status: str = "success",
|
||||
error_type: str | None = None,
|
||||
chunk_count: int = 0,
|
||||
duration_ms: int = 0,
|
||||
first_chunk_ms: int = -1,
|
||||
provider_completion_id: str | None = None,
|
||||
run_id: str | None = None,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
"""Best-effort truncation-event write from any loop/thread. Never raises."""
|
||||
global _store_cache, _store_cache_failed
|
||||
if _store_cache is None:
|
||||
if _store_cache_failed:
|
||||
return
|
||||
try:
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
session_factory = get_session_factory()
|
||||
except Exception:
|
||||
session_factory = None
|
||||
if session_factory is None:
|
||||
# No DB configured (memory backend); do not retry on every stream.
|
||||
_store_cache_failed = True
|
||||
return
|
||||
_store_cache = SqlStreamTruncationStore(session_factory)
|
||||
run_id_value = run_id if run_id and run_id not in {"none", ""} else None
|
||||
event = StreamTruncationEvent(
|
||||
id=uuid.uuid4().hex,
|
||||
created_at=datetime.now(UTC),
|
||||
model_name=model_name,
|
||||
provider_completion_id=provider_completion_id
|
||||
if provider_completion_id and provider_completion_id != "none"
|
||||
else None,
|
||||
run_id=run_id_value,
|
||||
event_type=event_type,
|
||||
strict_mode=strict_mode,
|
||||
finish_reason=finish_reason,
|
||||
status=status,
|
||||
error_type=error_type,
|
||||
chunk_count=chunk_count,
|
||||
duration_ms=duration_ms,
|
||||
first_chunk_ms=first_chunk_ms,
|
||||
detail=detail,
|
||||
)
|
||||
try:
|
||||
from deerflow.persistence.engine import fire_and_forget_db_write
|
||||
|
||||
fire_and_forget_db_write(_store_cache.record(event))
|
||||
except Exception:
|
||||
logger.debug("stream truncation event drop", exc_info=True)
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""Repository contract for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
# Hard retention cap. The user-facing promise is "最多存一个月" of truncation
|
||||
# records; the SQL repository enforces it on every insert so no scheduler is
|
||||
# required for the cap to hold.
|
||||
RETENTION_DAYS = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationEvent:
|
||||
"""In-memory shape of a truncation event before persistence."""
|
||||
|
||||
id: str
|
||||
created_at: datetime
|
||||
model_name: str | None = None
|
||||
provider_completion_id: str | None = None
|
||||
run_id: str | None = None
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: str = "silent_no_terminal"
|
||||
strict_mode: bool = False
|
||||
finish_reason: str = "none"
|
||||
status: str = "success"
|
||||
error_type: str | None = None
|
||||
chunk_count: int = 0
|
||||
duration_ms: int = 0
|
||||
first_chunk_ms: int = -1
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamTruncationQuery:
|
||||
"""Filter set for the admin list endpoint."""
|
||||
|
||||
model_name: str | None = None
|
||||
event_type: str | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
limit: int = 100
|
||||
offset: int = 0
|
||||
|
||||
|
||||
class StreamTruncationStore(ABC):
|
||||
"""Persistence contract for the model-layer recorder + admin API."""
|
||||
|
||||
@abstractmethod
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
"""Persist one event and enforce the retention cap. Must never raise."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Return matching rows (newest first), paginated by ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
"""Return the unpaginated row count for ``query``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
"""Per-model aggregate: ``[{model_name, event_type counts..., total, last_at}]``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
"""Delete rows older than ``cutoff``; return the deleted row count."""
|
||||
raise NotImplementedError
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
"""ORM model for model-stream truncation events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class StreamTruncationEventRow(Base):
|
||||
"""One row per detected model-stream truncation.
|
||||
|
||||
Written from the patched OpenAI-compatible streaming layer when a provider
|
||||
stream ends abnormally: without a terminal frame (silently committed half
|
||||
answer), via ``IncompleteChatStreamError`` (strict integrity mode, retried),
|
||||
on an output-length cap, or interrupted mid-stream by a transport error.
|
||||
Rows are retained for at most ``RETENTION_DAYS`` by the SQL repository.
|
||||
"""
|
||||
|
||||
__tablename__ = "stream_truncation_events"
|
||||
__table_args__ = (
|
||||
Index("ix_stream_truncation_created_model", "created_at", "model_name"),
|
||||
Index("ix_stream_truncation_created_event", "created_at", "event_type"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), index=True
|
||||
)
|
||||
|
||||
# Configured model name (the config.yaml entry name), sanitized by the model layer.
|
||||
model_name: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
# Provider-side completion id when the gateway supplied one; joins a row to
|
||||
# the upstream gateway's own request log.
|
||||
provider_completion_id: Mapped[str | None] = mapped_column(String(128))
|
||||
# LangGraph run id when the model layer could see one ("none" is dropped).
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
|
||||
# silent_no_terminal / incomplete_stream_error / length_truncated /
|
||||
# content_filter / midstream_error
|
||||
event_type: Mapped[str] = mapped_column(String(32), index=True)
|
||||
# Whether the model ran with strict_stream_integrity enabled.
|
||||
strict_mode: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
# Raw provider finish reason bucket: none / stop / length / tool_calls /
|
||||
# function_call / content_filter / end_turn / other.
|
||||
finish_reason: Mapped[str] = mapped_column(String(16), default="none")
|
||||
# Stream outcome from the model layer: success / error / cancelled.
|
||||
status: Mapped[str] = mapped_column(String(16), default="success")
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
chunk_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
duration_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
first_chunk_ms: Mapped[int] = mapped_column(Integer, default=-1)
|
||||
|
||||
detail: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat()
|
||||
if isinstance(self.created_at, datetime)
|
||||
else self.created_at,
|
||||
"model_name": self.model_name,
|
||||
"provider_completion_id": self.provider_completion_id,
|
||||
"run_id": self.run_id,
|
||||
"event_type": self.event_type,
|
||||
"strict_mode": bool(self.strict_mode),
|
||||
"finish_reason": self.finish_reason,
|
||||
"status": self.status,
|
||||
"error_type": self.error_type,
|
||||
"chunk_count": self.chunk_count,
|
||||
"duration_ms": self.duration_ms,
|
||||
"first_chunk_ms": self.first_chunk_ms,
|
||||
"detail": self.detail,
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
"""SQLAlchemy-backed stream-truncation event repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, desc, func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.stream_truncation.base import (
|
||||
RETENTION_DAYS,
|
||||
StreamTruncationEvent,
|
||||
StreamTruncationQuery,
|
||||
StreamTruncationStore,
|
||||
)
|
||||
from deerflow.persistence.stream_truncation.model import StreamTruncationEventRow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SqlStreamTruncationStore(StreamTruncationStore):
|
||||
"""Persist truncation events with a hard one-month retention cap."""
|
||||
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
async def _ensure_table(self, session) -> None:
|
||||
# Table creation follows the repo's create_all-on-first-use convention
|
||||
# (see e.g. task_buttons); safe to call repeatedly.
|
||||
try:
|
||||
await session.run_sync(
|
||||
lambda sync_session: StreamTruncationEventRow.__table__.create(
|
||||
sync_session.bind, checkfirst=True
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("stream_truncation table create skipped", exc_info=True)
|
||||
|
||||
async def record(self, event: StreamTruncationEvent) -> None:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
session.add(
|
||||
StreamTruncationEventRow(
|
||||
id=event.id,
|
||||
created_at=event.created_at,
|
||||
model_name=event.model_name,
|
||||
provider_completion_id=event.provider_completion_id,
|
||||
run_id=event.run_id,
|
||||
event_type=event.event_type,
|
||||
strict_mode=event.strict_mode,
|
||||
finish_reason=event.finish_reason,
|
||||
status=event.status,
|
||||
error_type=event.error_type,
|
||||
chunk_count=event.chunk_count,
|
||||
duration_ms=event.duration_ms,
|
||||
first_chunk_ms=event.first_chunk_ms,
|
||||
detail=event.detail,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
# Same session/transaction scope: enforce the retention cap on
|
||||
# every insert. Truncations are rare, so this stays cheap and
|
||||
# keeps the one-month promise without any scheduler.
|
||||
await self._enforce_retention(session)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to persist stream truncation event model=%s type=%s",
|
||||
event.model_name,
|
||||
event.event_type,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _enforce_retention(self, session) -> None:
|
||||
"""Delete rows beyond the retention cap. Best-effort, never raises."""
|
||||
try:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=RETENTION_DAYS)
|
||||
await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.debug("stream truncation retention sweep failed", exc_info=True)
|
||||
|
||||
def _apply_filters(self, query: StreamTruncationQuery):
|
||||
stmt = select(StreamTruncationEventRow)
|
||||
if query.model_name:
|
||||
stmt = stmt.where(StreamTruncationEventRow.model_name == query.model_name)
|
||||
if query.event_type:
|
||||
stmt = stmt.where(StreamTruncationEventRow.event_type == query.event_type)
|
||||
if query.since is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at >= query.since)
|
||||
if query.until is not None:
|
||||
stmt = stmt.where(StreamTruncationEventRow.created_at <= query.until)
|
||||
return stmt
|
||||
|
||||
async def list(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).order_by(
|
||||
desc(StreamTruncationEventRow.created_at)
|
||||
).limit(query.limit).offset(query.offset)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [row.to_dict() for row in rows]
|
||||
except Exception:
|
||||
logger.warning("Failed to list stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def count(self, query: StreamTruncationQuery) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
func.count(StreamTruncationEventRow.id)
|
||||
)
|
||||
return int((await session.execute(stmt)).scalar() or 0)
|
||||
except Exception:
|
||||
logger.warning("Failed to count stream truncation events", exc_info=True)
|
||||
return 0
|
||||
|
||||
async def model_breakdown(self, query: StreamTruncationQuery) -> list[dict[str, Any]]:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
stmt = self._apply_filters(query).with_only_columns(
|
||||
StreamTruncationEventRow.model_name,
|
||||
StreamTruncationEventRow.event_type,
|
||||
func.count(StreamTruncationEventRow.id).label("total"),
|
||||
func.max(StreamTruncationEventRow.created_at).label("last_at"),
|
||||
).group_by(StreamTruncationEventRow.model_name, StreamTruncationEventRow.event_type)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
merged: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
name = row.model_name or "(unknown)"
|
||||
bucket = merged.setdefault(
|
||||
name, {"model_name": name, "total": 0, "last_at": None}
|
||||
)
|
||||
bucket[row.event_type] = int(row.total or 0)
|
||||
bucket["total"] += int(row.total or 0)
|
||||
row_last = row.last_at
|
||||
if row_last is not None and (
|
||||
bucket["last_at"] is None or row_last > bucket["last_at"]
|
||||
):
|
||||
bucket["last_at"] = row_last
|
||||
result = sorted(
|
||||
merged.values(), key=lambda item: item["last_at"] or datetime.min.replace(tzinfo=UTC), reverse=True
|
||||
)
|
||||
for item in result:
|
||||
if isinstance(item["last_at"], datetime):
|
||||
item["last_at"] = item["last_at"].isoformat()
|
||||
return result
|
||||
except Exception:
|
||||
logger.warning("Failed to aggregate stream truncation events", exc_info=True)
|
||||
return []
|
||||
|
||||
async def purge_before(self, cutoff: datetime) -> int:
|
||||
try:
|
||||
async with self._session_factory() as session:
|
||||
await self._ensure_table(session)
|
||||
result = await session.execute(
|
||||
delete(StreamTruncationEventRow).where(
|
||||
StreamTruncationEventRow.created_at < cutoff
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
rowcount = result.rowcount
|
||||
return 0 if rowcount is None else int(rowcount)
|
||||
except Exception:
|
||||
logger.warning("Failed to purge stream truncation events", exc_info=True)
|
||||
return 0
|
||||
Reference in new issue
Block a user