sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -17,15 +17,25 @@ from datetime import UTC, datetime, timedelta
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from langchain_core.messages import AIMessage
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.admin_audit_retention import sweep_admin_audit_retention
|
||||
from deerflow.agents.middlewares.llm_metrics_middleware import (
|
||||
LlmMetricsMiddleware,
|
||||
_extract_usage,
|
||||
)
|
||||
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.llm_metrics import SqlLlmMetricsStore
|
||||
from deerflow.persistence.llm_metrics.base import LlmMetricRecord, LlmMetricsQuery
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.run import RunRepository
|
||||
from deerflow.persistence.run.model import RunRow
|
||||
from deerflow.persistence.tool_metrics import SqlToolMetricsStore
|
||||
from deerflow.persistence.tool_metrics.base import ToolMetricRecord, ToolMetricsQuery
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.types import BEIJING_TZ
|
||||
|
||||
|
||||
class _RecordingStore:
|
||||
@@ -52,10 +62,12 @@ class _RecordingStore:
|
||||
async def sql_store(tmp_path):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'metrics.db'}")
|
||||
async with engine.begin() as conn:
|
||||
# Importing the package registers the LlmCallMetricRow with Base.metadata.
|
||||
import deerflow.persistence.llm_metrics.model # noqa: F401
|
||||
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await conn.run_sync(
|
||||
lambda sync_conn: Base.metadata.create_all(
|
||||
sync_conn,
|
||||
tables=[LlmCallMetricRow.__table__],
|
||||
)
|
||||
)
|
||||
sf = async_sessionmaker(engine, expire_on_commit=False)
|
||||
store = SqlLlmMetricsStore(sf)
|
||||
try:
|
||||
@@ -64,6 +76,57 @@ async def sql_store(tmp_path):
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def tool_sql_store(tmp_path):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'tool-metrics.db'}")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
lambda sync_conn: Base.metadata.create_all(
|
||||
sync_conn,
|
||||
tables=[ToolCallMetricRow.__table__],
|
||||
)
|
||||
)
|
||||
store = SqlToolMetricsStore(async_sessionmaker(engine, expire_on_commit=False))
|
||||
try:
|
||||
yield store
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def run_sql_store(tmp_path):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'runs.db'}")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
lambda sync_conn: Base.metadata.create_all(
|
||||
sync_conn,
|
||||
tables=[RunRow.__table__],
|
||||
)
|
||||
)
|
||||
store = RunRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
try:
|
||||
yield store
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def snapshot_session_factory(tmp_path):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'snapshots.db'}")
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(
|
||||
lambda sync_conn: Base.metadata.create_all(
|
||||
sync_conn,
|
||||
tables=[AdminLeaderboardDailyStatRow.__table__],
|
||||
)
|
||||
)
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
yield session_factory
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _make_request():
|
||||
return type(
|
||||
"FakeRequest",
|
||||
@@ -258,6 +321,330 @@ async def test_sql_store_iter_all_paginates(sql_store):
|
||||
assert seen == 1200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_metric_purge_before_keeps_rows_at_or_after_cutoff(sql_store):
|
||||
"""The retention boundary is exclusive: a record exactly at cutoff stays."""
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=90)
|
||||
for record_id, created_at in (
|
||||
("old", cutoff - timedelta(days=1)),
|
||||
("boundary", cutoff),
|
||||
("fresh", cutoff + timedelta(days=1)),
|
||||
):
|
||||
await sql_store.record(
|
||||
LlmMetricRecord(id=record_id, created_at=created_at, duration_ms=1)
|
||||
)
|
||||
|
||||
assert await sql_store.purge_before(cutoff) == 1
|
||||
# A subsequent cleanup pass is idempotent.
|
||||
assert await sql_store.purge_before(cutoff) == 0
|
||||
|
||||
rows = await sql_store.list(LlmMetricsQuery(limit=10))
|
||||
assert {row["id"] for row in rows} == {"boundary", "fresh"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_metric_purge_before_keeps_rows_at_or_after_cutoff(tool_sql_store):
|
||||
"""Tool-call audit retention has the same strict-before-cutoff behavior."""
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=90)
|
||||
for record_id, created_at in (
|
||||
("old", cutoff - timedelta(days=1)),
|
||||
("boundary", cutoff),
|
||||
("fresh", cutoff + timedelta(days=1)),
|
||||
):
|
||||
await tool_sql_store.record(
|
||||
ToolMetricRecord(
|
||||
id=record_id,
|
||||
created_at=created_at,
|
||||
tool_name="web_search",
|
||||
duration_ms=1,
|
||||
)
|
||||
)
|
||||
|
||||
assert await tool_sql_store.purge_before(cutoff) == 1
|
||||
assert await tool_sql_store.purge_before(cutoff) == 0
|
||||
|
||||
rows = await tool_sql_store.list(ToolMetricsQuery(limit=10))
|
||||
assert {row["id"] for row in rows} == {"boundary", "fresh"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_run_summary_purge_keeps_boundary_and_live_runs(run_sql_store):
|
||||
"""Audit-summary retention never deletes a current, boundary, or late-completed run."""
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=90)
|
||||
async with run_sql_store._sf() as session:
|
||||
session.add_all(
|
||||
[
|
||||
RunRow(
|
||||
run_id="old-terminal",
|
||||
thread_id="thread-old-terminal",
|
||||
status="success",
|
||||
created_at=cutoff - timedelta(seconds=1),
|
||||
updated_at=cutoff - timedelta(seconds=1),
|
||||
),
|
||||
RunRow(
|
||||
run_id="boundary-terminal",
|
||||
thread_id="thread-boundary-terminal",
|
||||
status="error",
|
||||
created_at=cutoff,
|
||||
updated_at=cutoff,
|
||||
),
|
||||
RunRow(
|
||||
run_id="old-live",
|
||||
thread_id="thread-old-live",
|
||||
status="running",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
updated_at=cutoff - timedelta(days=1),
|
||||
),
|
||||
RunRow(
|
||||
run_id="old-created-recently-completed",
|
||||
thread_id="thread-old-created-recently-completed",
|
||||
status="success",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
updated_at=cutoff + timedelta(seconds=1),
|
||||
),
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
assert await run_sql_store.purge_terminal_before(cutoff) == 1
|
||||
assert await run_sql_store.get("old-terminal", user_id=None) is None
|
||||
assert await run_sql_store.get("boundary-terminal", user_id=None) is not None
|
||||
assert await run_sql_store.get("old-live", user_id=None) is not None
|
||||
assert await run_sql_store.get("old-created-recently-completed", user_id=None) is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_metric_purge_caps_one_large_batch_at_500(sql_store):
|
||||
"""A single metrics delete never expands into an unbounded SQL statement."""
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=90)
|
||||
async with sql_store._sf() as session:
|
||||
session.add_all(
|
||||
[
|
||||
LlmCallMetricRow(
|
||||
id=f"expired-{index}",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
duration_ms=1,
|
||||
)
|
||||
for index in range(501)
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# Even a much larger requested size must respect the portable 500-row cap.
|
||||
assert await sql_store.purge_before(cutoff, batch_size=10_000) == 500
|
||||
assert await sql_store.count(LlmMetricsQuery()) == 1
|
||||
assert await sql_store.purge_before(cutoff, batch_size=10_000) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_run_summary_purge_caps_one_large_batch_at_500(run_sql_store):
|
||||
"""Run-summary retention uses the same bounded primary-key deletion rule."""
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=90)
|
||||
async with run_sql_store._sf() as session:
|
||||
session.add_all(
|
||||
[
|
||||
RunRow(
|
||||
run_id=f"expired-run-{index}",
|
||||
thread_id=f"thread-{index}",
|
||||
status="success",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
updated_at=cutoff - timedelta(days=1),
|
||||
)
|
||||
for index in range(501)
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
assert await run_sql_store.purge_terminal_before(cutoff, batch_size=10_000) == 500
|
||||
async with run_sql_store._sf() as session:
|
||||
remaining = list((await session.scalars(select(RunRow.run_id))).all())
|
||||
assert len(remaining) == 1
|
||||
assert await run_sql_store.purge_terminal_before(cutoff, batch_size=10_000) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_retention_sweep_deletes_only_one_configured_batch_per_store(
|
||||
sql_store,
|
||||
run_sql_store,
|
||||
):
|
||||
"""A small sweep budget leaves the remaining historic backlog for later ticks."""
|
||||
|
||||
now = datetime(2026, 10, 26, tzinfo=UTC)
|
||||
cutoff = now - timedelta(days=90)
|
||||
async with sql_store._sf() as session:
|
||||
session.add_all(
|
||||
[
|
||||
LlmCallMetricRow(
|
||||
id=f"metric-{index}",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
duration_ms=1,
|
||||
)
|
||||
for index in range(11)
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
async with run_sql_store._sf() as session:
|
||||
session.add_all(
|
||||
[
|
||||
RunRow(
|
||||
run_id=f"run-{index}",
|
||||
thread_id=f"thread-{index}",
|
||||
status="success",
|
||||
created_at=cutoff - timedelta(days=1),
|
||||
updated_at=cutoff - timedelta(days=1),
|
||||
)
|
||||
for index in range(11)
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=None,
|
||||
llm_metrics_store=sql_store,
|
||||
run_store=run_sql_store,
|
||||
now=now,
|
||||
batch_size=10,
|
||||
max_batches_per_store=1,
|
||||
inter_batch_pause_seconds=0,
|
||||
max_sweep_seconds=60,
|
||||
)
|
||||
|
||||
assert result["llm_metrics"] == 10
|
||||
assert result["run_summaries"] == 10
|
||||
assert await sql_store.count(LlmMetricsQuery()) == 1
|
||||
async with run_sql_store._sf() as session:
|
||||
remaining_runs = list((await session.scalars(select(RunRow.run_id))).all())
|
||||
assert len(remaining_runs) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_retention_snapshot_batch_does_not_delete_an_entire_old_date(
|
||||
snapshot_session_factory,
|
||||
):
|
||||
"""Snapshot cleanup selects composite keys, not every settings hash for a date."""
|
||||
|
||||
now = datetime(2026, 10, 26, tzinfo=UTC)
|
||||
cutoff = now - timedelta(days=90)
|
||||
cutoff_date = cutoff.astimezone(BEIJING_TZ).date().isoformat()
|
||||
old_date = (cutoff - timedelta(days=1)).astimezone(BEIJING_TZ).date().isoformat()
|
||||
async with snapshot_session_factory() as session:
|
||||
session.add_all(
|
||||
[
|
||||
AdminLeaderboardDailyStatRow(
|
||||
stat_date=old_date,
|
||||
settings_hash=f"old-hash-{index:02d}",
|
||||
)
|
||||
for index in range(12)
|
||||
]
|
||||
+ [
|
||||
AdminLeaderboardDailyStatRow(
|
||||
stat_date=cutoff_date,
|
||||
settings_hash="boundary-hash",
|
||||
)
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=snapshot_session_factory,
|
||||
now=now,
|
||||
batch_size=10,
|
||||
max_batches_per_store=1,
|
||||
inter_batch_pause_seconds=0,
|
||||
max_sweep_seconds=60,
|
||||
)
|
||||
|
||||
assert result["leaderboard_snapshots"] == 10
|
||||
async with snapshot_session_factory() as session:
|
||||
rows = list((await session.scalars(select(AdminLeaderboardDailyStatRow))).all())
|
||||
assert sum(row.stat_date == old_date for row in rows) == 2
|
||||
assert [(row.stat_date, row.settings_hash) for row in rows if row.stat_date == cutoff_date] == [
|
||||
(cutoff_date, "boundary-hash")
|
||||
]
|
||||
|
||||
|
||||
class _AlwaysFullRetentionStore:
|
||||
"""A store that models an arbitrarily large historical backlog."""
|
||||
|
||||
def __init__(self, maximum_calls: int) -> None:
|
||||
self.calls: list[tuple[datetime, int]] = []
|
||||
self.maximum_calls = maximum_calls
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int) -> int:
|
||||
self.calls.append((cutoff, batch_size))
|
||||
# Fail fast if a future implementation loses the sweep budget instead
|
||||
# of letting this regression test loop forever.
|
||||
if len(self.calls) > self.maximum_calls:
|
||||
raise AssertionError("retention sweep exceeded its per-store work budget")
|
||||
return batch_size
|
||||
|
||||
|
||||
class _BusyRetentionStore:
|
||||
"""Models a table temporarily locked by another database writer."""
|
||||
|
||||
async def purge_before(self, cutoff: datetime, *, batch_size: int) -> int:
|
||||
raise RuntimeError("database is locked")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_retention_sweep_has_fixed_per_store_work_budget():
|
||||
"""One scheduler tick stops after a bounded amount of historic work."""
|
||||
|
||||
batch_size = 37
|
||||
max_batches = 3
|
||||
store = _AlwaysFullRetentionStore(maximum_calls=max_batches)
|
||||
now = datetime(2026, 10, 26, tzinfo=UTC)
|
||||
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=None,
|
||||
llm_metrics_store=store,
|
||||
now=now,
|
||||
batch_size=batch_size,
|
||||
max_batches_per_store=max_batches,
|
||||
inter_batch_pause_seconds=0,
|
||||
max_sweep_seconds=60,
|
||||
)
|
||||
|
||||
expected_cutoff = now - timedelta(days=90)
|
||||
assert store.calls == [(expected_cutoff, batch_size)] * max_batches
|
||||
assert result == {
|
||||
"llm_metrics": batch_size * max_batches,
|
||||
"tool_metrics": 0,
|
||||
"run_summaries": 0,
|
||||
"leaderboard_snapshots": 0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_retention_busy_store_does_not_block_other_sources():
|
||||
"""A lock on one audit table defers only that source to a later tick."""
|
||||
|
||||
available_store = _AlwaysFullRetentionStore(maximum_calls=1)
|
||||
result = await sweep_admin_audit_retention(
|
||||
session_factory=None,
|
||||
llm_metrics_store=_BusyRetentionStore(),
|
||||
tool_metrics_store=available_store,
|
||||
now=datetime(2026, 10, 26, tzinfo=UTC),
|
||||
batch_size=10,
|
||||
max_batches_per_store=1,
|
||||
inter_batch_pause_seconds=0,
|
||||
max_sweep_seconds=60,
|
||||
)
|
||||
|
||||
assert available_store.calls
|
||||
assert result == {
|
||||
"llm_metrics": 0,
|
||||
"tool_metrics": 10,
|
||||
"run_summaries": 0,
|
||||
"leaderboard_snapshots": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_usage_prefers_usage_metadata():
|
||||
response = AIMessage(
|
||||
content="x",
|
||||
|
||||
Reference in new issue
Block a user