fix: improve model concurrency limiting
This commit is contained in:
1 parent
aca089e8ef
commit
a046d19ce1
7 files changed
+162
-56
No files matched your search
@@ -232,6 +232,15 @@ function isActiveRunConflictError(message: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isModelConcurrencyLimitError(message: string): boolean {
|
||||
return (
|
||||
/\b429\b/.test(message) ||
|
||||
/并发已达到上限/.test(message) ||
|
||||
/使用人数较多/.test(message) ||
|
||||
/too many requests/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
function buildCurrentUserLabel(): string {
|
||||
const auth = getStoredAuth();
|
||||
if (!auth) {
|
||||
@@ -574,6 +583,10 @@ export function useThreadStream({
|
||||
toast.info("当前对话还在生成回答,请等待本轮回复结束,或先点「停止」再发送新的问题。");
|
||||
return;
|
||||
}
|
||||
if (isModelConcurrencyLimitError(message)) {
|
||||
toast.info("当前使用人数较多,请稍等一下再试。");
|
||||
return;
|
||||
}
|
||||
toast.error(message);
|
||||
},
|
||||
onFinish(state) {
|
||||
|
||||
@@ -85,7 +85,7 @@ export default function ModelConcurrencySettingsPage({
|
||||
className={
|
||||
embedded
|
||||
? "flex w-full flex-col gap-6"
|
||||
: "mx-auto flex w-full max-w-3xl flex-col gap-6 px-6 py-10"
|
||||
: "mx-auto flex h-full min-h-0 w-full max-w-3xl flex-col gap-6 overflow-y-auto px-6 py-10"
|
||||
}
|
||||
>
|
||||
<div>
|
||||
@@ -93,7 +93,7 @@ export default function ModelConcurrencySettingsPage({
|
||||
模型并发限制
|
||||
</h1>
|
||||
<p className="text-muted-foreground mt-1 text-sm">
|
||||
按模型限制同时进行中的问答数。达到上限后,新请求会收到“当前并发已达到上限,请稍后再试”的提示。
|
||||
按模型限制同时进行中的问答数。达到上限后,新请求会收到“当前使用人数较多,请稍等一下再试”的提示。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -21,13 +21,14 @@ from langchain_core.messages import HumanMessage
|
||||
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.agents_config import load_agent_config
|
||||
from deerflow.config.agents_config import load_agent_config, validate_agent_id
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.runtime import (
|
||||
END_SENTINEL,
|
||||
HEARTBEAT_SENTINEL,
|
||||
ConflictError,
|
||||
DisconnectMode,
|
||||
ModelConcurrencyLimitError,
|
||||
RunManager,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
@@ -180,9 +181,7 @@ def _rag_mode_requested(body: Any) -> bool:
|
||||
if isinstance(context, Mapping) and context.get("rag_mode_enabled"):
|
||||
return True
|
||||
metadata = getattr(body, "metadata", None)
|
||||
if isinstance(metadata, Mapping) and (
|
||||
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
|
||||
):
|
||||
if isinstance(metadata, Mapping) and (metadata.get("rag_mode_enabled") or metadata.get("rag_mode")):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -191,9 +190,7 @@ def _thread_rag_mode_enabled(record: Any) -> bool:
|
||||
if not isinstance(record, Mapping):
|
||||
return False
|
||||
metadata = record.get("metadata")
|
||||
return isinstance(metadata, Mapping) and bool(
|
||||
metadata.get("rag_mode_enabled") or metadata.get("rag_mode")
|
||||
)
|
||||
return isinstance(metadata, Mapping) and bool(metadata.get("rag_mode_enabled") or metadata.get("rag_mode"))
|
||||
|
||||
|
||||
def _append_invisible_prompt(graph_input: dict[str, Any], prompt: str) -> dict[str, Any]:
|
||||
@@ -420,6 +417,46 @@ def _extract_requested_model_name(body: Any) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _extract_effective_agent_id(body: Any) -> str | None:
|
||||
"""Best-effort read of the custom agent id that the lead graph will run."""
|
||||
candidates: list[str] = []
|
||||
|
||||
def collect(container: Mapping[str, Any] | None) -> None:
|
||||
if not container:
|
||||
return
|
||||
for key in ("agent_id", "agent_name"):
|
||||
value = str(container.get(key) or "").strip()
|
||||
if value:
|
||||
candidates.append(value)
|
||||
|
||||
context = getattr(body, "context", None)
|
||||
if isinstance(context, Mapping):
|
||||
collect(context)
|
||||
|
||||
config = getattr(body, "config", None)
|
||||
if isinstance(config, Mapping):
|
||||
nested_context = config.get("context")
|
||||
nested_configurable = config.get("configurable")
|
||||
if isinstance(nested_context, Mapping):
|
||||
collect(nested_context)
|
||||
if isinstance(nested_configurable, Mapping):
|
||||
collect(nested_configurable)
|
||||
|
||||
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
|
||||
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
candidates.append(assistant_id)
|
||||
|
||||
for agent_id in candidates:
|
||||
try:
|
||||
normalized = validate_agent_id(agent_id)
|
||||
except ValueError:
|
||||
logger.debug("Ignoring invalid agent id during model resolution: %s", agent_id)
|
||||
continue
|
||||
if normalized:
|
||||
return normalized
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_effective_run_model_name(body: Any) -> str:
|
||||
"""Resolve the model that this run will effectively use for concurrency limiting."""
|
||||
app_config = get_app_config()
|
||||
@@ -438,14 +475,14 @@ def _resolve_effective_run_model_name(body: Any) -> str:
|
||||
)
|
||||
return default_model_name
|
||||
|
||||
assistant_id = str(getattr(body, "assistant_id", "") or "").strip()
|
||||
if assistant_id and assistant_id not in _BUILTIN_GRAPH_IDS:
|
||||
agent_id = _extract_effective_agent_id(body)
|
||||
if agent_id:
|
||||
try:
|
||||
agent_config = load_agent_config(assistant_id)
|
||||
agent_config = load_agent_config(agent_id)
|
||||
except FileNotFoundError:
|
||||
agent_config = None
|
||||
except Exception:
|
||||
logger.debug("Failed to load agent config for model resolution: %s", assistant_id, exc_info=True)
|
||||
logger.debug("Failed to load agent config for model resolution: %s", agent_id, exc_info=True)
|
||||
agent_config = None
|
||||
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
|
||||
if agent_model_name:
|
||||
@@ -454,37 +491,25 @@ def _resolve_effective_run_model_name(body: Any) -> str:
|
||||
logger.warning(
|
||||
"Agent model '%s' for agent '%s' not found; falling back to default model '%s'.",
|
||||
agent_model_name,
|
||||
assistant_id,
|
||||
agent_id,
|
||||
default_model_name,
|
||||
)
|
||||
|
||||
return default_model_name
|
||||
|
||||
|
||||
async def _enforce_model_concurrency_limit(run_mgr: RunManager, model_name: str) -> None:
|
||||
"""Reject the run when the selected model has reached its concurrent-run limit."""
|
||||
def _get_model_concurrency_limit(model_name: str) -> int | None:
|
||||
"""Return the configured limit for a model, or ``None`` when limiting is off."""
|
||||
settings = load_system_settings().model_concurrency
|
||||
if not settings.enabled:
|
||||
return
|
||||
return None
|
||||
|
||||
limit = settings.model_limits.get(model_name, settings.default_limit)
|
||||
try:
|
||||
limit = int(limit)
|
||||
except (TypeError, ValueError):
|
||||
limit = int(settings.default_limit)
|
||||
limit = max(1, limit)
|
||||
|
||||
active = await run_mgr.list_active()
|
||||
current = sum(
|
||||
1
|
||||
for record in active
|
||||
if str((record.metadata or {}).get("model_name") or "").strip() == model_name
|
||||
)
|
||||
if current >= limit:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"当前模型 {model_name} 并发已达到上限({limit}),请稍后再试。",
|
||||
)
|
||||
return max(1, limit)
|
||||
|
||||
|
||||
async def _sync_legacy_agent_records(store) -> None:
|
||||
@@ -663,7 +688,7 @@ async def start_run(
|
||||
t_post_enforce = time.perf_counter()
|
||||
|
||||
effective_model_name = _resolve_effective_run_model_name(body)
|
||||
await _enforce_model_concurrency_limit(run_mgr, effective_model_name)
|
||||
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
|
||||
|
||||
run_metadata = dict(body.metadata or {})
|
||||
run_metadata["model_name"] = effective_model_name
|
||||
@@ -676,7 +701,10 @@ async def start_run(
|
||||
metadata=run_metadata,
|
||||
kwargs={"input": body.input, "config": body.config},
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if model_concurrency_limit is not None else None,
|
||||
)
|
||||
except ModelConcurrencyLimitError as exc:
|
||||
raise HTTPException(status_code=429, detail="当前使用人数较多,请稍等一下再试。") from exc
|
||||
except ConflictError as exc:
|
||||
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
|
||||
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
|
||||
@@ -792,8 +820,7 @@ async def start_run(
|
||||
# common cause is a stale ``_maybe_sync_legacy_agent_records`` TTL
|
||||
# window firing on a machine with hundreds of agent directories.
|
||||
logger.warning(
|
||||
"[start-run-timing] thread=%s total=%.3fs enforce_access=%.3fs "
|
||||
"run_create=%.3fs thread_upsert=%.3fs build_payload=%.3fs",
|
||||
"[start-run-timing] thread=%s total=%.3fs enforce_access=%.3fs run_create=%.3fs thread_upsert=%.3fs build_payload=%.3fs",
|
||||
sanitize_log_param(thread_id),
|
||||
total,
|
||||
t_post_enforce - t_pre_enforce,
|
||||
|
||||
@@ -6,7 +6,17 @@ directly from ``deerflow.runtime``.
|
||||
"""
|
||||
|
||||
from .checkpointer import checkpointer_context, get_checkpointer, make_checkpointer, reset_checkpointer
|
||||
from .runs import ConflictError, DisconnectMode, RunContext, RunManager, RunRecord, RunStatus, UnsupportedStrategyError, run_agent
|
||||
from .runs import (
|
||||
ConflictError,
|
||||
DisconnectMode,
|
||||
ModelConcurrencyLimitError,
|
||||
RunContext,
|
||||
RunManager,
|
||||
RunRecord,
|
||||
RunStatus,
|
||||
UnsupportedStrategyError,
|
||||
run_agent,
|
||||
)
|
||||
from .serialization import serialize, serialize_channel_values, serialize_lc_object, serialize_messages_tuple
|
||||
from .store import get_store, make_store, reset_store, store_context
|
||||
from .stream_bridge import END_SENTINEL, HEARTBEAT_SENTINEL, MemoryStreamBridge, StreamBridge, StreamEvent, make_stream_bridge
|
||||
@@ -20,6 +30,7 @@ __all__ = [
|
||||
# runs
|
||||
"ConflictError",
|
||||
"DisconnectMode",
|
||||
"ModelConcurrencyLimitError",
|
||||
"RunContext",
|
||||
"RunManager",
|
||||
"RunRecord",
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
"""Run lifecycle management for LangGraph Platform API compatibility."""
|
||||
|
||||
from .manager import ConflictError, RunManager, RunRecord, UnsupportedStrategyError
|
||||
from .manager import ConflictError, ModelConcurrencyLimitError, RunManager, RunRecord, UnsupportedStrategyError
|
||||
from .schemas import DisconnectMode, RunStatus
|
||||
from .worker import RunContext, run_agent
|
||||
|
||||
__all__ = [
|
||||
"ConflictError",
|
||||
"DisconnectMode",
|
||||
"ModelConcurrencyLimitError",
|
||||
"RunContext",
|
||||
"RunManager",
|
||||
"RunRecord",
|
||||
|
||||
@@ -217,6 +217,7 @@ class RunManager:
|
||||
metadata: dict | None = None,
|
||||
kwargs: dict | None = None,
|
||||
multitask_strategy: str = "reject",
|
||||
model_concurrency_limit: tuple[str, int] | None = None,
|
||||
) -> RunRecord:
|
||||
"""Atomically check for inflight runs and create a new one.
|
||||
|
||||
@@ -256,6 +257,12 @@ class RunManager:
|
||||
multitask_strategy,
|
||||
)
|
||||
|
||||
if model_concurrency_limit is not None:
|
||||
model_name, limit = model_concurrency_limit
|
||||
active_for_model = sum(1 for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running) and str((r.metadata or {}).get("model_name") or "").strip() == model_name)
|
||||
if active_for_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit)
|
||||
|
||||
record = RunRecord(
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
@@ -306,5 +313,14 @@ class ConflictError(Exception):
|
||||
"""Raised when multitask_strategy=reject and thread has inflight runs."""
|
||||
|
||||
|
||||
class ModelConcurrencyLimitError(Exception):
|
||||
"""Raised when a model-specific in-flight run limit has been reached."""
|
||||
|
||||
def __init__(self, model_name: str, limit: int) -> None:
|
||||
self.model_name = model_name
|
||||
self.limit = limit
|
||||
super().__init__(f"Model {model_name} has reached its concurrency limit")
|
||||
|
||||
|
||||
class UnsupportedStrategyError(Exception):
|
||||
"""Raised when a multitask_strategy value is not yet implemented."""
|
||||
@@ -7,15 +7,14 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.routers import system_settings as ss_router
|
||||
from app.gateway.services import (
|
||||
_enforce_model_concurrency_limit,
|
||||
_get_model_concurrency_limit,
|
||||
_resolve_effective_run_model_name,
|
||||
)
|
||||
from deerflow.config import system_settings as ss
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
from deerflow.runtime.runs.manager import ModelConcurrencyLimitError, RunManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -77,9 +76,7 @@ def test_resolve_effective_run_model_name_prefers_requested_model(
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name}
|
||||
if name in {"default-model", "glm5", "agent-model"}
|
||||
else None,
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "glm5", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
@@ -100,9 +97,7 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name}
|
||||
if name in {"default-model", "agent-model"}
|
||||
else None,
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
@@ -118,8 +113,50 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
|
||||
assert _resolve_effective_run_model_name(body) == "agent-model"
|
||||
|
||||
|
||||
def test_resolve_effective_run_model_name_uses_context_agent_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
"app.gateway.services.load_agent_config",
|
||||
lambda agent_id: SimpleNamespace(model="agent-model") if agent_id == "custom-agent" else SimpleNamespace(model="default-model"),
|
||||
)
|
||||
|
||||
body = SimpleNamespace(
|
||||
assistant_id="lead_agent",
|
||||
context={"agent_id": "custom-agent"},
|
||||
config=None,
|
||||
)
|
||||
assert _resolve_effective_run_model_name(body) == "agent-model"
|
||||
|
||||
|
||||
def test_resolve_effective_run_model_name_context_agent_overrides_assistant_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "context-model", "assistant-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
"app.gateway.services.load_agent_config",
|
||||
lambda agent_id: SimpleNamespace(model="context-model" if agent_id == "context-agent" else "assistant-model"),
|
||||
)
|
||||
|
||||
body = SimpleNamespace(
|
||||
assistant_id="assistant-agent",
|
||||
context={"agent_id": "context-agent"},
|
||||
config=None,
|
||||
)
|
||||
assert _resolve_effective_run_model_name(body) == "context-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
|
||||
async def test_run_manager_rejects_model_limit_atomically(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
@@ -132,18 +169,21 @@ async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
|
||||
)
|
||||
),
|
||||
)
|
||||
assert _get_model_concurrency_limit("glm5") == 2
|
||||
|
||||
run_mgr = RunManager()
|
||||
await run_mgr.create("t1", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create("t2", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create_or_reject("t1", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create_or_reject("t2", metadata={"model_name": "glm5"})
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await _enforce_model_concurrency_limit(run_mgr, "glm5")
|
||||
assert exc.value.status_code == 429
|
||||
assert "并发已达到上限" in str(exc.value.detail)
|
||||
with pytest.raises(ModelConcurrencyLimitError):
|
||||
await run_mgr.create_or_reject(
|
||||
"t3",
|
||||
metadata={"model_name": "glm5"},
|
||||
model_concurrency_limit=("glm5", 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enforce_model_concurrency_limit_skips_when_disabled(
|
||||
def test_get_model_concurrency_limit_skips_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
@@ -156,6 +196,4 @@ async def test_enforce_model_concurrency_limit_skips_when_disabled(
|
||||
)
|
||||
),
|
||||
)
|
||||
run_mgr = RunManager()
|
||||
await run_mgr.create("t1", metadata={"model_name": "glm5"})
|
||||
await _enforce_model_concurrency_limit(run_mgr, "glm5")
|
||||
assert _get_model_concurrency_limit("glm5") is None
|
||||
Reference in new issue
Block a user