fix: improve model concurrency limiting
This commit is contained in:
1 parent
aca089e8ef
commit
a046d19ce1
7 files changed
+1266
-1160
No files matched your search
@@ -7,15 +7,14 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.routers import system_settings as ss_router
|
||||
from app.gateway.services import (
|
||||
_enforce_model_concurrency_limit,
|
||||
_get_model_concurrency_limit,
|
||||
_resolve_effective_run_model_name,
|
||||
)
|
||||
from deerflow.config import system_settings as ss
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
from deerflow.runtime.runs.manager import ModelConcurrencyLimitError, RunManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -77,9 +76,7 @@ def test_resolve_effective_run_model_name_prefers_requested_model(
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name}
|
||||
if name in {"default-model", "glm5", "agent-model"}
|
||||
else None,
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "glm5", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
@@ -100,9 +97,7 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name}
|
||||
if name in {"default-model", "agent-model"}
|
||||
else None,
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
@@ -118,8 +113,50 @@ def test_resolve_effective_run_model_name_uses_agent_default_when_request_missin
|
||||
assert _resolve_effective_run_model_name(body) == "agent-model"
|
||||
|
||||
|
||||
def test_resolve_effective_run_model_name_uses_context_agent_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "agent-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
"app.gateway.services.load_agent_config",
|
||||
lambda agent_id: SimpleNamespace(model="agent-model") if agent_id == "custom-agent" else SimpleNamespace(model="default-model"),
|
||||
)
|
||||
|
||||
body = SimpleNamespace(
|
||||
assistant_id="lead_agent",
|
||||
context={"agent_id": "custom-agent"},
|
||||
config=None,
|
||||
)
|
||||
assert _resolve_effective_run_model_name(body) == "agent-model"
|
||||
|
||||
|
||||
def test_resolve_effective_run_model_name_context_agent_overrides_assistant_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
app_config = SimpleNamespace(
|
||||
models=[SimpleNamespace(name="default-model")],
|
||||
get_model_config=lambda name: {"name": name} if name in {"default-model", "context-model", "assistant-model"} else None,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.services.get_app_config", lambda: app_config)
|
||||
monkeypatch.setattr(
|
||||
"app.gateway.services.load_agent_config",
|
||||
lambda agent_id: SimpleNamespace(model="context-model" if agent_id == "context-agent" else "assistant-model"),
|
||||
)
|
||||
|
||||
body = SimpleNamespace(
|
||||
assistant_id="assistant-agent",
|
||||
context={"agent_id": "context-agent"},
|
||||
config=None,
|
||||
)
|
||||
assert _resolve_effective_run_model_name(body) == "context-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
|
||||
async def test_run_manager_rejects_model_limit_atomically(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
@@ -132,18 +169,21 @@ async def test_enforce_model_concurrency_limit_rejects_when_limit_reached(
|
||||
)
|
||||
),
|
||||
)
|
||||
assert _get_model_concurrency_limit("glm5") == 2
|
||||
|
||||
run_mgr = RunManager()
|
||||
await run_mgr.create("t1", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create("t2", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create_or_reject("t1", metadata={"model_name": "glm5"})
|
||||
await run_mgr.create_or_reject("t2", metadata={"model_name": "glm5"})
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await _enforce_model_concurrency_limit(run_mgr, "glm5")
|
||||
assert exc.value.status_code == 429
|
||||
assert "并发已达到上限" in str(exc.value.detail)
|
||||
with pytest.raises(ModelConcurrencyLimitError):
|
||||
await run_mgr.create_or_reject(
|
||||
"t3",
|
||||
metadata={"model_name": "glm5"},
|
||||
model_concurrency_limit=("glm5", 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enforce_model_concurrency_limit_skips_when_disabled(
|
||||
def test_get_model_concurrency_limit_skips_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
@@ -156,6 +196,4 @@ async def test_enforce_model_concurrency_limit_skips_when_disabled(
|
||||
)
|
||||
),
|
||||
)
|
||||
run_mgr = RunManager()
|
||||
await run_mgr.create("t1", metadata={"model_name": "glm5"})
|
||||
await _enforce_model_concurrency_limit(run_mgr, "glm5")
|
||||
assert _get_model_concurrency_limit("glm5") is None
|
||||
Reference in new issue
Block a user