feat: add per-model concurrency limits
This commit is contained in:
1 parent
24eb629773
commit
aca089e8ef
11 files changed
+695
-25
No files matched your search
@@ -15,13 +15,15 @@ import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.runtime import (
|
||||
from fastapi import HTTPException, Request
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.runtime import (
|
||||
END_SENTINEL,
|
||||
HEARTBEAT_SENTINEL,
|
||||
ConflictError,
|
||||
@@ -366,7 +368,7 @@ def build_run_config(
|
||||
return config
|
||||
|
||||
|
||||
def _collect_requested_agent_ids(body: Any) -> set[str]:
|
||||
def _collect_requested_agent_ids(body: Any) -> set[str]:
|
||||
"""Collect agent ids from assistant_id/config/context for access checks."""
|
||||
agent_ids: set[str] = set()
|
||||
if body.assistant_id and body.assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
@@ -393,10 +395,99 @@ def _collect_requested_agent_ids(body: Any) -> set[str]:
|
||||
if isinstance(nested_configurable, Mapping):
|
||||
collect(nested_configurable)
|
||||
|
||||
return agent_ids
|
||||
|
||||
|
||||
async def _sync_legacy_agent_records(store) -> None:
|
||||
return agent_ids
|
||||
|
||||
|
||||
def _extract_requested_model_name(body: Any) -> str | None:
|
||||
"""Best-effort read of the model override requested for this run."""
|
||||
context = getattr(body, "context", None)
|
||||
if isinstance(context, Mapping):
|
||||
for key in ("model_name", "model"):
|
||||
value = str(context.get(key) or "").strip()
|
||||
if value:
|
||||
return value
|
||||
|
||||
config = getattr(body, "config", None)
|
||||
if isinstance(config, Mapping):
|
||||
for container_key in ("context", "configurable"):
|
||||
container = config.get(container_key)
|
||||
if not isinstance(container, Mapping):
|
||||
continue
|
||||
for key in ("model_name", "model"):
|
||||
value = str(container.get(key) or "").strip()
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_effective_run_model_name(body: Any) -> str:
|
||||
"""Resolve the model that this run will effectively use for concurrency limiting."""
|
||||
app_config = get_app_config()
|
||||
default_model_name = app_config.models[0].name if app_config.models else None
|
||||
if not default_model_name:
|
||||
raise ValueError("No chat models are configured. Please configure at least one model in config.yaml.")
|
||||
|
||||
requested_model_name = _extract_requested_model_name(body)
|
||||
if requested_model_name:
|
||||
if app_config.get_model_config(requested_model_name):
|
||||
return requested_model_name
|
||||
logger.warning(
|
||||
"Requested model '%s' not found; falling back to default model '%s'.",
|
||||
requested_model_name,
|
||||
default_model_name,
|
||||
)
|
||||
return default_model_name
|
||||
|
||||
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
|
||||
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
try:
|
||||
agent_config = load_agent_config(assistant_id)
|
||||
except FileNotFoundError:
|
||||
agent_config = None
|
||||
except Exception:
|
||||
logger.debug("Failed to load agent config for model resolution: %s", assistant_id, exc_info=True)
|
||||
agent_config = None
|
||||
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
|
||||
if agent_model_name:
|
||||
if app_config.get_model_config(agent_model_name):
|
||||
return agent_model_name
|
||||
logger.warning(
|
||||
"Agent model '%s' for agent '%s' not found; falling back to default model '%s'.",
|
||||
agent_model_name,
|
||||
assistant_id,
|
||||
default_model_name,
|
||||
)
|
||||
|
||||
return default_model_name
|
||||
|
||||
|
||||
async def _enforce_model_concurrency_limit(run_mgr: RunManager, model_name: str) -> None:
|
||||
"""Reject the run when the selected model has reached its concurrent-run limit."""
|
||||
settings = load_system_settings().model_concurrency
|
||||
if not settings.enabled:
|
||||
return
|
||||
|
||||
limit = settings.model_limits.get(model_name, settings.default_limit)
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError):
|
||||
limit = int(settings.default_limit)
|
||||
limit = max(1, limit)
|
||||
|
||||
active = await run_mgr.list_active()
|
||||
current = sum(
|
||||
1
|
||||
for record in active
|
||||
if str((record.metadata or {}).get("model_name") or "").strip() == model_name
|
||||
)
|
||||
if current >= limit:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"当前模型 {model_name} 并发已达到上限({limit}),请稍后再试。",
|
||||
)
|
||||
|
||||
|
||||
async def _sync_legacy_agent_records(store) -> None:
|
||||
"""Import existing folder-backed agents as built-in records.
|
||||
|
||||
Heavy: ``list_custom_agents()`` scans ``.deer-flow/agents/`` and reads
|
||||
@@ -567,19 +658,25 @@ async def start_run(
|
||||
# admin-configured mapping rather than from a per-user-visible assistant_id.
|
||||
# Those run under the "default" bucket, where a user-owned unpublished agent
|
||||
# would otherwise 404 the access check.
|
||||
if enforce_access:
|
||||
await _enforce_agent_access(body, request)
|
||||
t_post_enforce = time.perf_counter()
|
||||
|
||||
try:
|
||||
record = await run_mgr.create_or_reject(
|
||||
thread_id,
|
||||
body.assistant_id,
|
||||
on_disconnect=disconnect,
|
||||
metadata=body.metadata or {},
|
||||
kwargs={"input": body.input, "config": body.config},
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
)
|
||||
if enforce_access:
|
||||
await _enforce_agent_access(body, request)
|
||||
t_post_enforce = time.perf_counter()
|
||||
|
||||
effective_model_name = _resolve_effective_run_model_name(body)
|
||||
await _enforce_model_concurrency_limit(run_mgr, effective_model_name)
|
||||
|
||||
run_metadata = dict(body.metadata or {})
|
||||
run_metadata["model_name"] = effective_model_name
|
||||
|
||||
try:
|
||||
record = await run_mgr.create_or_reject(
|
||||
thread_id,
|
||||
body.assistant_id,
|
||||
on_disconnect=disconnect,
|
||||
metadata=run_metadata,
|
||||
kwargs={"input": body.input, "config": body.config},
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
)
|
||||
except ConflictError as exc:
|
||||
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
|
||||
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
|
||||
|
||||
Reference in new issue
Block a user