fix: improve model concurrency limiting

This commit is contained in:
Administrator committed 2026-06-27 18:57:01 +08:00
1 parent aca089e8ef
commit a046d19ce1
7 files changed
+162 -56

No files matched your search

+13
View File
@@ -232,6 +232,15 @@ function isActiveRunConflictError(message: string): boolean {
);
}
function isModelConcurrencyLimitError(message: string): boolean {
return (
/\b429\b/.test(message) ||
/并发已达到上限/.test(message) ||
/使用人数较多/.test(message) ||
/too many requests/i.test(message)
);
}
function buildCurrentUserLabel(): string {
const auth = getStoredAuth();
if (!auth) {
@@ -574,6 +583,10 @@ export function useThreadStream({
toast.info("当前对话还在生成回答,请等待本轮回复结束,或先点「停止」再发送新的问题。");
return;
}
if (isModelConcurrencyLimitError(message)) {
toast.info("当前使用人数较多,请稍等一下再试。");
return;
}
toast.error(message);
},
onFinish(state) {
@@ -85,7 +85,7 @@ export default function ModelConcurrencySettingsPage({
className={
embedded
? "flex w-full flex-col gap-6"
: "mx-auto flex w-full max-w-3xl flex-col gap-6 px-6 py-10"
: "mx-auto flex h-full min-h-0 w-full max-w-3xl flex-col gap-6 overflow-y-auto px-6 py-10"
}
>
<div>
@@ -93,7 +93,7 @@ export default function ModelConcurrencySettingsPage({
模型并发限制
</h1>
<p className="text-muted-foreground mt-1 text-sm">
按模型限制同时进行中的问答数。达到上限后,新请求会收到“当前并发已达到上限,请稍后再试”的提示。
按模型限制同时进行中的问答数。达到上限后,新请求会收到“当前使用人数较多,请稍等一下再试”的提示。
</p>
</div>
@@ -21,13 +21,14 @@ from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
from deerflow.config.agents_config import load_agent_config, validate_agent_id
from deerflow.config.system_settings import load_system_settings
from deerflow.runtime import (
END_SENTINEL,
HEARTBEAT_SENTINEL,
ConflictError,
DisconnectMode,
ModelConcurrencyLimitError,
RunManager,
RunRecord,
RunStatus,
@@ -180,9 +181,7 @@ def _rag_mode_requested(body: Any) -> bool:
if isinstance(context, Mapping) and context.get("rag_mode_enabled"):
return True
metadata = getattr(body, "metadata", None)
if isinstance(metadata, Mapping) and (
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
):
if isinstance(metadata, Mapping) and (metadata.get("rag_mode_enabled") or metadata.get("rag_mode")):
return True
return False
@@ -191,9 +190,7 @@ def _thread_rag_mode_enabled(record: Any) -> bool:
if not isinstance(record, Mapping):
return False
metadata = record.get("metadata")
return isinstance(metadata, Mapping) and bool(
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
)
return isinstance(metadata, Mapping) and bool(metadata.get("rag_mode_enabled") or metadata.get("rag_mode"))
def _append_invisible_prompt(graph_input: dict[str, Any], prompt: str) -> dict[str, Any]:
@@ -420,6 +417,46 @@ def _extract_requested_model_name(body: Any) -> str | None:
return None
def _extract_effective_agent_id(body: Any) -> str | None:
"""Best-effort read of the custom agent id that the lead graph will run."""
candidates: list[str] = []
def collect(container: Mapping[str, Any] | None) -> None:
if not container:
return
for key in ("agent_id", "agent_name"):
value = str(container.get(key) or "").strip()
if value:
candidates.append(value)
context = getattr(body, "context", None)
if isinstance(context, Mapping):
collect(context)
config = getattr(body, "config", None)
if isinstance(config, Mapping):
nested_context = config.get("context")
nested_configurable = config.get("configurable")
if isinstance(nested_context, Mapping):
collect(nested_context)
if isinstance(nested_configurable, Mapping):
collect(nested_configurable)
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
candidates.append(assistant_id)
for agent_id in candidates:
try:
normalized = validate_agent_id(agent_id)
except ValueError:
logger.debug("Ignoring invalid agent id during model resolution: %s", agent_id)
continue
if normalized:
return normalized
return None
def _resolve_effective_run_model_name(body: Any) -> str:
"""Resolve the model that this run will effectively use for concurrency limiting."""
app_config = get_app_config()
@@ -438,14 +475,14 @@ def _resolve_effective_run_model_name(body: Any) -> str:
)
return default_model_name
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
agent_id = _extract_effective_agent_id(body)
if agent_id:
try:
agent_config = load_agent_config(assistant_id)
agent_config = load_agent_config(agent_id)
except FileNotFoundError:
agent_config = None
except Exception:
logger.debug("Failed to load agent config for model resolution: %s", assistant_id, exc_info=True)
logger.debug("Failed to load agent config for model resolution: %s", agent_id, exc_info=True)
agent_config = None
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
if agent_model_name:
@@ -454,37 +491,25 @@ def _resolve_effective_run_model_name(body: Any) -> str:
logger.warning(
"Agent model '%s' for agent '%s' not found; falling back to default model '%s'.",
agent_model_name,
assistant_id,
agent_id,
default_model_name,
)
return default_model_name
async def _enforce_model_concurrency_limit(run_mgr: RunManager, model_name: str) -> None:
"""Reject the run when the selected model has reached its concurrent-run limit."""
def _get_model_concurrency_limit(model_name: str) -> int | None:
"""Return the configured limit for a model, or ``None`` when limiting is off."""
settings = load_system_settings().model_concurrency
if not settings.enabled:
return
return None
limit = settings.model_limits.get(model_name, settings.default_limit)
try:
limit = int(limit)
except (TypeError, ValueError):
limit = int(settings.default_limit)
limit = max(1, limit)
active = await run_mgr.list_active()
current = sum(
1
for record in active
if str((record.metadata or {}).get("model_name") or "").strip() == model_name
)
if current >= limit:
raise HTTPException(
status_code=429,
detail=f"当前模型 {model_name} 并发已达到上限({limit}),请稍后再试。",
)
return max(1, limit)
async def _sync_legacy_agent_records(store) -> None:
@@ -663,7 +688,7 @@ async def start_run(
t_post_enforce = time.perf_counter()
effective_model_name = _resolve_effective_run_model_name(body)
await _enforce_model_concurrency_limit(run_mgr, effective_model_name)
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
@@ -676,7 +701,10 @@ async def start_run(
metadata=run_metadata,
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if model_concurrency_limit is not None else None,
)
except ModelConcurrencyLimitError as exc:
raise HTTPException(status_code=429, detail="当前使用人数较多,请稍等一下再试。") from exc
except ConflictError as exc:
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
@@ -792,8 +820,7 @@ async def start_run(
# common cause is a stale ``_maybe_sync_legacy_agent_records`` TTL
# window firing on a machine with hundreds of agent directories.
logger.warning(
"[start-run-timing] thread=%s total=%.3fs enforce_access=%.3fs "
"run_create=%.3fs thread_upsert=%.3fs build_payload=%.3fs",
"[start-run-timing] thread=%s total=%.3fs enforce_access=%.3fs run_create=%.3fs thread_upsert=%.3fs build_payload=%.3fs",
sanitize_log_param(thread_id),
total,
t_post_enforce - t_pre_enforce,
@@ -6,7 +6,17 @@ directly from ``deerflow.runtime``.
"""
from .checkpointer import checkpointer_context, get_checkpointer, make_checkpointer, reset_checkpointer
from .runs import ConflictError, DisconnectMode, RunContext, RunManager, RunRecord, RunStatus, UnsupportedStrategyError, run_agent
from .runs import (
ConflictError,
DisconnectMode,
ModelConcurrencyLimitError,
RunContext,
RunManager,
RunRecord,
RunStatus,
UnsupportedStrategyError,
run_agent,
)
from .serialization import serialize, serialize_channel_values, serialize_lc_object, serialize_messages_tuple
from .store import get_store, make_store, reset_store, store_context
from .stream_bridge import END_SENTINEL, HEARTBEAT_SENTINEL, MemoryStreamBridge, StreamBridge, StreamEvent, make_stream_bridge
@@ -20,6 +30,7 @@ __all__ = [
# runs
"ConflictError",
"DisconnectMode",
"ModelConcurrencyLimitError",
"RunContext",
"RunManager",
"RunRecord",
@@ -1,12 +1,13 @@
"""Run lifecycle management for LangGraph Platform API compatibility."""
from .manager import ConflictError, RunManager, RunRecord, UnsupportedStrategyError
from .manager import ConflictError, ModelConcurrencyLimitError, RunManager, RunRecord, UnsupportedStrategyError
from .schemas import DisconnectMode, RunStatus
from .worker import RunContext, run_agent
__all__ = [
"ConflictError",
"DisconnectMode",
"ModelConcurrencyLimitError",
"RunContext",
"RunManager",
"RunRecord",
@@ -217,6 +217,7 @@ class RunManager:
metadata: dict | None = None,
kwargs: dict | None = None,
multitask_strategy: str = "reject",
model_concurrency_limit: tuple[str, int] | None = None,
) -> RunRecord:
"""Atomically check for inflight runs and create a new one.
@@ -256,6 +257,12 @@ class RunManager:
multitask_strategy,
)
if model_concurrency_limit is not None:
model_name, limit = model_concurrency_limit
active_for_model = sum(1 for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running) and str((r.metadata or {}).get("model_name") or "").strip() == model_name)
if active_for_model >= max(1, int(limit)):
raise ModelConcurrencyLimitError(model_name, limit)
record = RunRecord(
run_id=run_id,
thread_id=thread_id,
@@ -306,5 +313,14 @@ class ConflictError(Exception):
"""Raised when multitask_strategy=reject and thread has inflight runs."""
class ModelConcurrencyLimitError(Exception):
"""Raised when a model-specific in-flight run limit has been reached."""
def __init__(self, model_name: str, limit: int) -> None:
self.model_name = model_name
self.limit = limit
super().__init__(f"Model {model_name} has reached its concurrency limit")
class UnsupportedStrategyError(Exception):
"""Raised when a multitask_strategy value is not yet implemented."""
@@ -7,15 +7,14 @@ from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import HTTPException
from app.gateway.routers import system_settings as ss_router
from app.gateway.services import (
_enforce_model_concurrency_limit,
_get_model_concurrency_limit,
_resolve_effective_run_model_name,
)
from deerflow.config import system_settings as ss
from deerflow.runtime.runs.manager import RunManager
from deerflow.runtime.runs.manager import ModelConcurrencyLimitError, RunManager
@pytest.fixture
@@ -77,9 +76,7 @@ def test_resolve_effective_run_model_name_prefers_requested_model(
) -> None:
app_config = SimpleNamespace(
models=[SimpleNamespace(name="default-model")],
get_model_config=lambda name: {"name": name}
if name in {"default-model", "glm5", "agent-model"}
else None,
get_model_config=lambda name: {"name": name} if name in {"default-model", "glm5", "agent-model"} else None,
)
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
monkeypatch.setattr(
@@ -100,9 +97,7 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
) -> None:
app_config = SimpleNamespace(
models=[SimpleNamespace(name="default-model")],
get_model_config=lambda name: {"name": name}
if name in {"default-model", "agent-model"}
else None,
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
)
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
monkeypatch.setattr(
@@ -118,8 +113,50 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
assert _resolve_effective_run_model_name(body) == "agent-model"
def test_resolve_effective_run_model_name_uses_context_agent_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
app_config = SimpleNamespace(
models=[SimpleNamespace(name="default-model")],
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
)
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
monkeypatch.setattr(
"app.gateway.services.load_agent_config",
lambda agent_id: SimpleNamespace(model="agent-model") if agent_id == "custom-agent" else SimpleNamespace(model="default-model"),
)
body = SimpleNamespace(
assistant_id="lead_agent",
context={"agent_id": "custom-agent"},
config=None,
)
assert _resolve_effective_run_model_name(body) == "agent-model"
def test_resolve_effective_run_model_name_context_agent_overrides_assistant_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
app_config = SimpleNamespace(
models=[SimpleNamespace(name="default-model")],
get_model_config=lambda name: {"name": name} if name in {"default-model", "context-model", "assistant-model"} else None,
)
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
monkeypatch.setattr(
"app.gateway.services.load_agent_config",
lambda agent_id: SimpleNamespace(model="context-model" if agent_id == "context-agent" else "assistant-model"),
)
body = SimpleNamespace(
assistant_id="assistant-agent",
context={"agent_id": "context-agent"},
config=None,
)
assert _resolve_effective_run_model_name(body) == "context-model"
@pytest.mark.asyncio
async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
async def test_run_manager_rejects_model_limit_atomically(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
@@ -132,18 +169,21 @@ async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
)
),
)
assert _get_model_concurrency_limit("glm5") == 2
run_mgr = RunManager()
await run_mgr.create("t1", metadata={"model_name": "glm5"})
await run_mgr.create("t2", metadata={"model_name": "glm5"})
await run_mgr.create_or_reject("t1", metadata={"model_name": "glm5"})
await run_mgr.create_or_reject("t2", metadata={"model_name": "glm5"})
with pytest.raises(HTTPException) as exc:
await _enforce_model_concurrency_limit(run_mgr, "glm5")
assert exc.value.status_code == 429
assert "并发已达到上限" in str(exc.value.detail)
with pytest.raises(ModelConcurrencyLimitError):
await run_mgr.create_or_reject(
"t3",
metadata={"model_name": "glm5"},
model_concurrency_limit=("glm5", 2),
)
@pytest.mark.asyncio
async def test_enforce_model_concurrency_limit_skips_when_disabled(
def test_get_model_concurrency_limit_skips_when_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
@@ -156,6 +196,4 @@ async def test_enforce_model_concurrency_limit_skips_when_disabled(
)
),
)
run_mgr = RunManager()
await run_mgr.create("t1", metadata={"model_name": "glm5"})
await _enforce_model_concurrency_limit(run_mgr, "glm5")
assert _get_model_concurrency_limit("glm5") is None