feat(监控): 管理员排行榜实时并发监控 + 真正停止对话 + 409 中文化

- 排行榜页顶部新增「实时并发监控」面板(admin-only):当前并发大模型调用数 +
  调用列表(用户/智能体/类型/已运行时长),每行「停止」+「全部停止」经
  RunManager.cancel 真正中断 LLM 流式调用,减轻大模型压力
- 并发量折线图(峰值/均值,按分钟/小时):新增轻量后台采样器写入新表
  concurrency_samples(create_all 自动建表),可按分/时聚合
- 大模型出字速度折线图(tokens/秒,按分钟/小时):复用 llm_call_metrics 聚合,
  便于定位是模型/供应商慢
- 后台采样可开关、默认关闭(opt-in)、运行时切换免重启、对线上零影响
  (单条极小 INSERT、保留期清理、查询有上限、全程 best-effort)
- 409 中文化:上一轮未结束又发新问题时返回中文提示,前端 onError 友好映射

后端:runtime/runs/manager.py(list_active)、persistence/concurrency(新模块)、
config/system_settings(ConcurrencyMonitorSettings)、gateway/routers/admin_active_runs、
app.py(采样后台任务)、services.py(409)。前端:core/admin/{active-runs,hooks}、
components/workspace/admin/concurrency-monitor、AdminUserLeaderboardPage、threads/hooks。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
吉鑫升andClaude Opus 4.8 committed 2026-06-26 20:29:26 +08:00
1 parent 158d157af4
commit 26f8010a57
17 files changed
+1488 -1

No files matched your search

@@ -27,6 +27,7 @@ from app.gateway.csrf_middleware import CSRFMiddleware
from app.gateway.deps import langgraph_runtime
from app.gateway.routers import (
ai_writing,
admin_active_runs,
admin_users,
agents,
artifacts,
@@ -488,6 +489,18 @@ _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
"ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
60.0,
)
# Concurrency (system-pressure) sampler: how often to snapshot the in-flight run
# count, and how long to keep samples (drives the admin 并发量 line chart, which
# aggregates per minute/hour). Sampling is in-memory cheap (one tiny INSERT).
_CONCURRENCY_SAMPLE_ENABLED = _env_bool("CONCURRENCY_SAMPLE_ENABLED", True)
_CONCURRENCY_SAMPLE_INTERVAL_SECONDS = _env_float(
"CONCURRENCY_SAMPLE_INTERVAL_SECONDS",
15.0,
)
_CONCURRENCY_SAMPLE_RETENTION_HOURS = _env_float(
"CONCURRENCY_SAMPLE_RETENTION_HOURS",
168.0, # 7 days
)
# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
# *previous* day's snapshot once, so records that landed near midnight are fully
# captured. Default 1 ? "???? 1 ????????".
@@ -828,6 +841,58 @@ async def _leaderboard_snapshot_scheduler_loop() -> None:
_release_singleton_lock(lock_handle)
async def _concurrency_sampler_loop(app: FastAPI) -> None:
"""Periodically record the in-flight run count for the admin pressure chart.
Runs in the gateway process, singleton-locked so only one worker samples.
Each tick records ``len(run_manager.list_active())`` and, roughly hourly,
purges samples older than the retention window. Best-effort throughout — a
failed tick is logged and the loop continues.
"""
if not _CONCURRENCY_SAMPLE_ENABLED:
logger.info("Concurrency sampler disabled by CONCURRENCY_SAMPLE_ENABLED")
return
store = getattr(app.state, "concurrency_sample_store", None)
run_mgr = getattr(app.state, "run_manager", None)
if store is None or run_mgr is None:
logger.info("Concurrency sampler not started (no store / run manager — memory backend?)")
return
from deerflow.config.runtime_paths import runtime_home
lock_handle = _acquire_singleton_lock(runtime_home() / ".concurrency_sampler.lock")
if lock_handle is None:
logger.info("Concurrency sampler: another gateway worker holds the lock; this worker skips")
return
interval = max(1.0, _CONCURRENCY_SAMPLE_INTERVAL_SECONDS)
purge_every_ticks = max(1, int(3600 / interval)) # ~once per hour
logger.info("Concurrency sampler started: interval=%.1fs retention=%.1fh", interval, _CONCURRENCY_SAMPLE_RETENTION_HOURS)
ticks = 0
try:
while True:
try:
# Runtime toggle (admin can turn sampling off without a restart).
# mtime-cached read, so this is cheap every tick.
from deerflow.config.system_settings import get_concurrency_monitor_settings
if get_concurrency_monitor_settings().enabled:
active = await run_mgr.list_active()
await store.record(len(active))
ticks += 1
if ticks % purge_every_ticks == 0:
from datetime import UTC, datetime, timedelta
cutoff = datetime.now(UTC) - timedelta(hours=_CONCURRENCY_SAMPLE_RETENTION_HOURS)
await store.purge_older_than(cutoff)
except Exception:
logger.exception("Concurrency sampler tick failed (non-fatal)")
await asyncio.sleep(interval)
finally:
_release_singleton_lock(lock_handle)
def _warn_if_multi_worker() -> None:
"""?? uvicorn / gunicorn ? worker ???? ERROR ???
@@ -1044,6 +1109,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
# Concurrency (system-pressure) sampler — lightweight, singleton-locked,
# runtime-toggleable via system_settings.concurrency_monitor.enabled.
try:
app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
except Exception:
logger.exception("Concurrency sampler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
@@ -1102,6 +1174,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Leaderboard snapshot scheduler shutdown error")
concurrency_sampler_task = getattr(app.state, "concurrency_sampler_task", None)
if concurrency_sampler_task is not None:
concurrency_sampler_task.cancel()
try:
await concurrency_sampler_task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("Concurrency sampler shutdown error")
# ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
# thread ??? LangGraph Server ? checkpointer ???lifespan ???
# ``langgraph_runtime()`` ? contextmanager ???????????????
@@ -1434,6 +1516,8 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
app.include_router(admin_users.router)
app.include_router(admin_users.self_router)
app.include_router(admin_active_runs.router)
# Admin tool / skill call metrics audit + export
app.include_router(tool_metrics.router)