feat: add per-model concurrency limits

This commit is contained in:
Administrator committed 2026-06-27 18:57:01 +08:00
1 parent 24eb629773
commit aca089e8ef
11 files changed
+695 -25

No files matched your search

@@ -15,13 +15,15 @@ import time
from collections.abc import Mapping
from typing import Any
from fastapi import HTTPException, Request
from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config.system_settings import load_system_settings
from deerflow.runtime import (
from fastapi import HTTPException, Request
from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
from deerflow.config.system_settings import load_system_settings
from deerflow.runtime import (
END_SENTINEL,
HEARTBEAT_SENTINEL,
ConflictError,
@@ -366,7 +368,7 @@ def build_run_config(
return config
def _collect_requested_agent_ids(body: Any) -> set[str]:
def _collect_requested_agent_ids(body: Any) -> set[str]:
"""Collect agent ids from assistant_id/config/context for access checks."""
agent_ids: set[str] = set()
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
@@ -393,10 +395,99 @@ def _collect_requested_agent_ids(body: Any) -> set[str]:
if isinstance(nested_configurable, Mapping):
collect(nested_configurable)
return agent_ids
async def _sync_legacy_agent_records(store) -> None:
return agent_ids
def _extract_requested_model_name(body: Any) -> str | None:
"""Best-effort read of the model override requested for this run."""
context = getattr(body, "context", None)
if isinstance(context, Mapping):
for key in ("model_name", "model"):
value = str(context.get(key) or "").strip()
if value:
return value
config = getattr(body, "config", None)
if isinstance(config, Mapping):
for container_key in ("context", "configurable"):
container = config.get(container_key)
if not isinstance(container, Mapping):
continue
for key in ("model_name", "model"):
value = str(container.get(key) or "").strip()
if value:
return value
return None
def _resolve_effective_run_model_name(body: Any) -> str:
"""Resolve the model that this run will effectively use for concurrency limiting."""
app_config = get_app_config()
default_model_name = app_config.models[0].name if app_config.models else None
if not default_model_name:
raise ValueError("No chat models are configured. Please configure at least one model in config.yaml.")
requested_model_name = _extract_requested_model_name(body)
if requested_model_name:
if app_config.get_model_config(requested_model_name):
return requested_model_name
logger.warning(
"Requested model '%s' not found; falling back to default model '%s'.",
requested_model_name,
default_model_name,
)
return default_model_name
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
try:
agent_config = load_agent_config(assistant_id)
except FileNotFoundError:
agent_config = None
except Exception:
logger.debug("Failed to load agent config for model resolution: %s", assistant_id, exc_info=True)
agent_config = None
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
if agent_model_name:
if app_config.get_model_config(agent_model_name):
return agent_model_name
logger.warning(
"Agent model '%s' for agent '%s' not found; falling back to default model '%s'.",
agent_model_name,
assistant_id,
default_model_name,
)
return default_model_name
async def _enforce_model_concurrency_limit(run_mgr: RunManager, model_name: str) -> None:
"""Reject the run when the selected model has reached its concurrent-run limit."""
settings = load_system_settings().model_concurrency
if not settings.enabled:
return
limit = settings.model_limits.get(model_name, settings.default_limit)
try:
limit = int(limit)
except (TypeError, ValueError):
limit = int(settings.default_limit)
limit = max(1, limit)
active = await run_mgr.list_active()
current = sum(
1
for record in active
if str((record.metadata or {}).get("model_name") or "").strip() == model_name
)
if current >= limit:
raise HTTPException(
status_code=429,
detail=f"当前模型 {model_name} 并发已达到上限({limit}),请稍后再试。",
)
async def _sync_legacy_agent_records(store) -> None:
"""Import existing folder-backed agents as built-in records.
Heavy: ``list_custom_agents()`` scans ``.deer-flow/agents/`` and reads
@@ -567,19 +658,25 @@ async def start_run(
# admin-configured mapping rather than from a per-user-visible assistant_id.
# Those run under the "default" bucket, where a user-owned unpublished agent
# would otherwise 404 the access check.
if enforce_access:
await _enforce_agent_access(body, request)
t_post_enforce = time.perf_counter()
try:
record = await run_mgr.create_or_reject(
thread_id,
body.assistant_id,
on_disconnect=disconnect,
metadata=body.metadata or {},
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
)
if enforce_access:
await _enforce_agent_access(body, request)
t_post_enforce = time.perf_counter()
effective_model_name = _resolve_effective_run_model_name(body)
await _enforce_model_concurrency_limit(run_mgr, effective_model_name)
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
try:
record = await run_mgr.create_or_reject(
thread_id,
body.assistant_id,
on_disconnect=disconnect,
metadata=run_metadata,
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
)
except ConflictError as exc:
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。