feat(监控): 管理员排行榜实时并发监控 + 真正停止对话 + 409 中文化

- 排行榜页顶部新增「实时并发监控」面板(admin-only):当前并发大模型调用数 +
  调用列表(用户/智能体/类型/已运行时长),每行「停止」+「全部停止」经
  RunManager.cancel 真正中断 LLM 流式调用,减轻大模型压力
- 并发量折线图(峰值/均值,按分钟/小时):新增轻量后台采样器写入新表
  concurrency_samples(create_all 自动建表),可按分/时聚合
- 大模型出字速度折线图(tokens/秒,按分钟/小时):复用 llm_call_metrics 聚合,
  便于定位是模型/供应商慢
- 后台采样可开关、默认关闭(opt-in)、运行时切换免重启、对线上零影响
  (单条极小 INSERT、保留期清理、查询有上限、全程 best-effort)
- 409 中文化:上一轮未结束又发新问题时返回中文提示,前端 onError 友好映射

后端:runtime/runs/manager.py(list_active)、persistence/concurrency(新模块)、
config/system_settings(ConcurrencyMonitorSettings)、gateway/routers/admin_active_runs、
app.py(采样后台任务)、services.py(409)。前端:core/admin/{active-runs,hooks}、
components/workspace/admin/concurrency-monitor、AdminUserLeaderboardPage、threads/hooks。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.8 committed 2026-06-26 20:29:26 +08:00
1 parent 158d157af4
commit 26f8010a57
17 files changed
+1488 -1

No files matched your search

@@ -234,6 +234,21 @@ class LoginWhitelistSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class ConcurrencyMonitorSettings(BaseModel):
"""管理员「实时并发监控」开关(排行榜页的系统压力监控)。
``enabled`` 控制**后台并发采样器**是否持续把当前并发调用数写库(供并发量
折线图按分钟/小时聚合)。**默认关**(opt-in,由管理员在排行榜页手动开启),
以对线上服务零影响为先;开启后采样器才开始写库,关闭则立即停止(每 tick 都
重读该开关,免重启),实时快照查询与「停止」始终可用、零常驻成本。设计上即便
开启也对线上零影响:采样仅一条极小 INSERT、有保留期清理、聚合查询有行数上限、
全程 best-effort。
"""
enabled: bool = Field(default=False, description="是否启用后台并发采样(默认关;开→开始写库,关→停止写库、折线图不再新增历史点)")
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -246,6 +261,7 @@ class SystemSettings(BaseModel):
leaderboard: LeaderboardSettings = Field(default_factory=LeaderboardSettings)
skill_compression: SkillCompressionSettings = Field(default_factory=SkillCompressionSettings)
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_config = ConfigDict(extra="ignore")
@@ -280,6 +296,15 @@ def get_skill_compression_settings() -> SkillCompressionSettings:
return SkillCompressionSettings()
def get_concurrency_monitor_settings() -> ConcurrencyMonitorSettings:
"""Convenience accessor for the concurrency-monitor block (safe defaults on error)."""
try:
return load_system_settings().concurrency_monitor
except Exception:
logger.debug("Failed to read concurrency monitor settings; using defaults", exc_info=True)
return ConcurrencyMonitorSettings()
def save_system_settings(settings: SystemSettings) -> SystemSettings:
"""Persist the settings document atomically, returning the saved value."""
path = _settings_path()
@@ -0,0 +1,15 @@
"""Concurrency (system-pressure) sampling persistence."""
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.concurrency.store import (
ConcurrencyBucket,
ConcurrencySampleStore,
make_concurrency_sample_store,
)
__all__ = [
"ConcurrencyBucket",
"ConcurrencySampleRow",
"ConcurrencySampleStore",
"make_concurrency_sample_store",
]
@@ -0,0 +1,31 @@
"""ORM model for periodic concurrency (system-pressure) samples.
A background sampler writes one row every few seconds holding the number of
in-flight model runs at that instant. The admin monitor aggregates these into
per-minute / per-hour buckets to draw the 并发量 pressure line chart, even for
time windows before the admin opened the page.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ConcurrencySampleRow(Base):
"""One sampled snapshot of concurrent in-flight model runs."""
__tablename__ = "concurrency_samples"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# Wall-clock time the sample was taken (Beijing time on disk, tz-aware in Python).
sampled_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), index=True, default=lambda: datetime.now(UTC)
)
# Number of runs whose status was pending/running at ``sampled_at``.
active_count: Mapped[int] = mapped_column(Integer, default=0)
@@ -0,0 +1,142 @@
"""Async store for concurrency samples + minute/hour aggregation."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
logger = logging.getLogger(__name__)
# How far back the aggregation will fetch raw rows before bucketing. A safety
# cap so a degenerate range never tries to load the whole table into memory.
_MAX_ROWS = 60_000
@dataclass
class ConcurrencyBucket:
"""One minute/hour bucket of the concurrency series."""
bucket: str # ISO start of the bucket
peak: int # max concurrent runs observed in the bucket
avg: float # mean concurrent runs in the bucket
samples: int # how many raw samples fell into the bucket
def _bucket_start(dt: datetime, granularity: str) -> datetime:
"""Truncate a timestamp to the start of its minute/hour bucket."""
if granularity == "hour":
return dt.replace(minute=0, second=0, microsecond=0)
return dt.replace(second=0, microsecond=0)
class ConcurrencySampleStore:
"""Persists periodic concurrency snapshots and aggregates them."""
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def record(self, active_count: int, *, sampled_at: datetime | None = None) -> None:
"""Insert one sample. Best-effort — never raises into the caller."""
try:
async with self._sf() as session:
session.add(
ConcurrencySampleRow(
active_count=max(0, int(active_count)),
sampled_at=sampled_at or datetime.now(UTC),
)
)
await session.commit()
except Exception: # pragma: no cover - sampling must never break the app
logger.warning("Failed to record concurrency sample", exc_info=True)
async def aggregate(
self,
*,
since: datetime,
until: datetime,
granularity: str = "minute",
) -> list[ConcurrencyBucket]:
"""Return per-minute / per-hour buckets of peak + average concurrency.
Bucketing is done in Python (not SQL ``date_trunc``/``strftime``) so the
logic is identical across the sqlite/mysql/postgres backends.
"""
gran = "hour" if granularity == "hour" else "minute"
try:
async with self._sf() as session:
rows = (
await session.execute(
select(ConcurrencySampleRow.sampled_at, ConcurrencySampleRow.active_count)
.where(
ConcurrencySampleRow.sampled_at >= since,
ConcurrencySampleRow.sampled_at <= until,
)
.order_by(ConcurrencySampleRow.sampled_at)
.limit(_MAX_ROWS)
)
).all()
except Exception:
logger.warning("Failed to aggregate concurrency samples", exc_info=True)
return []
buckets: dict[datetime, dict[str, float]] = {}
for sampled_at, active_count in rows:
if sampled_at is None:
continue
if sampled_at.tzinfo is None:
sampled_at = sampled_at.replace(tzinfo=UTC)
key = _bucket_start(sampled_at, gran)
value = int(active_count or 0)
entry = buckets.get(key)
if entry is None:
buckets[key] = {"peak": value, "sum": value, "count": 1}
else:
entry["peak"] = max(entry["peak"], value)
entry["sum"] += value
entry["count"] += 1
result: list[ConcurrencyBucket] = []
for key in sorted(buckets):
entry = buckets[key]
count = max(1, int(entry["count"]))
result.append(
ConcurrencyBucket(
bucket=key.isoformat(),
peak=int(entry["peak"]),
avg=round(entry["sum"] / count, 2),
samples=count,
)
)
return result
async def purge_older_than(self, cutoff: datetime) -> int:
"""Delete samples older than ``cutoff``. Returns rows removed."""
try:
async with self._sf() as session:
result = await session.execute(
delete(ConcurrencySampleRow).where(ConcurrencySampleRow.sampled_at < cutoff)
)
await session.commit()
return int(result.rowcount or 0)
except Exception: # pragma: no cover
logger.warning("Failed to purge concurrency samples", exc_info=True)
return 0
def make_concurrency_sample_store(
session_factory: async_sessionmaker[AsyncSession] | None,
) -> ConcurrencySampleStore | None:
"""Factory mirroring the other ``make_*_store`` helpers.
Returns ``None`` on the memory backend (no SQL session factory), in which
case the monitor simply reports a live snapshot with no historical chart.
"""
if session_factory is None:
return None
return ConcurrencySampleStore(session_factory)
@@ -23,6 +23,7 @@ from deerflow.persistence.agents.model import (
)
from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.feedback.model import FeedbackRow
@@ -85,6 +86,7 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
"KnowledgeEmbeddingRow",
"KnowledgeEntityRow",
@@ -279,6 +279,20 @@ class RunManager:
async with self._lock:
return any(r.thread_id == thread_id and r.status in (RunStatus.pending, RunStatus.running) for r in self._runs.values())
async def list_active(self) -> list[RunRecord]:
"""Return every in-flight run (status pending/running), oldest first.
Powers the admin concurrency monitor: each record represents one live
call into the model runtime (interactive chat, agent chat, roundtable
seat, …), so ``len(list_active())`` is the current concurrent-call count
and each record can be cancelled to relieve model pressure. Completed
records linger in ``_runs`` until :meth:`cleanup`, so we filter by status
rather than returning the whole registry.
"""
async with self._lock:
# Dict insertion order == creation order, so this is oldest-first.
return [r for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running)]
async def cleanup(self, run_id: str, *, delay: float = 300) -> None:
"""Remove a run record after an optional delay."""
if delay > 0: