Add Redis-backed concurrency gate

This commit is contained in:
吉鑫升 committed 2026-06-30 11:25:44 +08:00
1 parent b0db8cc3d8
commit b4206d6421
12 files changed
+654 -46

No files matched your search

@@ -12,13 +12,14 @@ import json
import logging
import re
import time
import uuid
from collections.abc import Mapping
from typing import Any
from fastapi import HTTPException, Request
from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.deps import get_concurrency_gate, get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config, validate_agent_id
@@ -41,6 +42,45 @@ from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
async def _run_agent_with_concurrency_lease(
*,
concurrency_gate: Any | None,
lease_acquired: bool,
lease_metadata: dict[str, str],
heartbeat_interval_seconds: int,
run_agent_kwargs: dict[str, Any],
) -> None:
"""Run the agent while refreshing and finally releasing an optional lease."""
heartbeat_task: asyncio.Task | None = None
async def _heartbeat_loop() -> None:
if concurrency_gate is None or not lease_acquired:
return
interval = max(5, int(heartbeat_interval_seconds or 20))
while True:
await asyncio.sleep(interval)
await concurrency_gate.heartbeat(**lease_metadata)
if concurrency_gate is not None and lease_acquired:
heartbeat_task = asyncio.create_task(
_heartbeat_loop(),
name=f"concurrency-heartbeat:{lease_metadata.get('run_id')}",
)
try:
await run_agent(**run_agent_kwargs)
finally:
if heartbeat_task is not None:
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
if concurrency_gate is not None and lease_acquired:
await concurrency_gate.release(**lease_metadata)
# ---------------------------------------------------------------------------
# SSE formatting
# ---------------------------------------------------------------------------
@@ -737,6 +777,7 @@ async def start_run(
bridge = get_stream_bridge(request)
run_mgr = get_run_manager(request)
run_ctx = get_run_context(request)
concurrency_gate = get_concurrency_gate(request)
disconnect = DisconnectMode.cancel if body.on_disconnect == "cancel" else DisconnectMode.continue_
@@ -754,11 +795,46 @@ async def start_run(
effective_user_id = get_effective_user_id()
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
user_model_concurrency_limit = _get_user_model_concurrency_limit(effective_model_name)
run_id = str(uuid.uuid4())
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
run_metadata["user_id"] = effective_user_id
redis_lease_acquired = False
redis_gate_degraded = True
lease_metadata = {
"run_id": run_id,
"thread_id": thread_id,
"model_name": effective_model_name,
"user_id": effective_user_id,
}
if concurrency_gate is not None:
acquire = await concurrency_gate.try_acquire(
run_id=run_id,
thread_id=thread_id,
model_name=effective_model_name,
user_id=effective_user_id,
assistant_id=body.assistant_id or "",
model_limit=model_concurrency_limit,
user_model_limit=user_model_concurrency_limit,
reject_thread=body.multitask_strategy == "reject",
)
redis_lease_acquired = acquire.acquired
redis_gate_degraded = acquire.degraded
if acquire.rejected:
if acquire.reason == "thread":
raise HTTPException(
status_code=409,
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
)
if acquire.reason == "user_model":
raise HTTPException(
status_code=429,
detail="This account has reached the concurrency limit for the selected model. Please try again later.",
)
raise HTTPException(status_code=429, detail="The selected model is busy. Please try again later.")
try:
record = await run_mgr.create_or_reject(
thread_id,
@@ -767,24 +843,35 @@ async def start_run(
metadata=run_metadata,
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if model_concurrency_limit is not None else None,
user_model_concurrency_limit=(effective_model_name, effective_user_id, user_model_concurrency_limit) if user_model_concurrency_limit is not None else None,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if redis_gate_degraded and model_concurrency_limit is not None else None,
user_model_concurrency_limit=(effective_model_name, effective_user_id, user_model_concurrency_limit) if redis_gate_degraded and user_model_concurrency_limit is not None else None,
run_id=run_id,
)
except ModelConcurrencyLimitError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
if getattr(exc, "scope", "") == "user_model":
detail = "当前账号对该模型的并发调用已达到上限,请等待已有回答结束后再试。"
detail = "This account has reached the concurrency limit for the selected model. Please try again later."
else:
detail = "当前使用人数较多,请稍等一下再试。"
detail = "The selected model is busy. Please try again later."
raise HTTPException(status_code=429, detail=detail) from exc
except ConflictError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
raise HTTPException(
status_code=409,
detail="当前对话还在生成回答,请等待本轮回复结束,或先点「停止」再发送新的问题。",
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
) from exc
except UnsupportedStrategyError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise HTTPException(status_code=501, detail=str(exc)) from exc
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
t_post_create = time.perf_counter()
# Upsert thread metadata so the thread appears in /threads/search,
@@ -822,7 +909,12 @@ async def start_run(
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
t_post_thread_upsert = time.perf_counter()
agent_factory = resolve_agent_factory(body.assistant_id)
try:
agent_factory = resolve_agent_factory(body.assistant_id)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
graph_input = normalize_input(body.input)
# Admin-configurable prompt prefix: when the caller opted in via
@@ -860,24 +952,38 @@ async def start_run(
stream_modes = normalize_stream_modes(body.stream_mode)
t_pre_task = time.perf_counter()
task = asyncio.create_task(
run_agent(
bridge,
run_mgr,
record,
ctx=run_ctx,
agent_factory=agent_factory,
graph_input=graph_input,
config=config,
stream_modes=stream_modes,
stream_subgraphs=body.stream_subgraphs,
interrupt_before=body.interrupt_before,
interrupt_after=body.interrupt_after,
# 透传 SDK ``Command``:resume / update / goto。Pregel 会据此从最近
# checkpoint 处恢复,而不是把 graph_input 当新一轮输入跑。
command=getattr(body, "command", None),
heartbeat_interval_seconds = 20
if concurrency_gate is not None:
heartbeat_interval_seconds = int(
getattr(getattr(concurrency_gate, "config", None), "heartbeat_interval_seconds", 20) or 20
)
)
try:
task = asyncio.create_task(
_run_agent_with_concurrency_lease(
concurrency_gate=concurrency_gate,
lease_acquired=redis_lease_acquired,
lease_metadata=lease_metadata,
heartbeat_interval_seconds=heartbeat_interval_seconds,
run_agent_kwargs={
"bridge": bridge,
"run_manager": run_mgr,
"record": record,
"ctx": run_ctx,
"agent_factory": agent_factory,
"graph_input": graph_input,
"config": config,
"stream_modes": stream_modes,
"stream_subgraphs": body.stream_subgraphs,
"interrupt_before": body.interrupt_before,
"interrupt_after": body.interrupt_after,
"command": getattr(body, "command", None),
},
)
)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
record.task = task
# Title sync is handled by worker.py's finally block which reads the