并发版
This commit is contained in:
1 parent
cd2ca406a5
commit
d39b58255d
11 files changed
+530
-73
No files matched your search
@@ -286,6 +286,16 @@ class ModelConcurrencySettings(BaseModel):
|
||||
default_factory=dict,
|
||||
description="Optional per-model overrides: model_name -> max concurrent runs.",
|
||||
)
|
||||
user_default_limit: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
le=10_000,
|
||||
description="Fallback max concurrent runs per user and model. 0 disables the per-user fallback.",
|
||||
)
|
||||
user_model_limits: dict[str, int] = Field(
|
||||
default_factory=dict,
|
||||
description="Optional per-user per-model overrides: model_name -> max concurrent runs for one user.",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
@@ -218,6 +218,7 @@ class RunManager:
|
||||
kwargs: dict | None = None,
|
||||
multitask_strategy: str = "reject",
|
||||
model_concurrency_limit: tuple[str, int] | None = None,
|
||||
user_model_concurrency_limit: tuple[str, str, int] | None = None,
|
||||
) -> RunRecord:
|
||||
"""Atomically check for inflight runs and create a new one.
|
||||
|
||||
@@ -261,7 +262,19 @@ class RunManager:
|
||||
model_name, limit = model_concurrency_limit
|
||||
active_for_model = sum(1 for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running) and str((r.metadata or {}).get("model_name") or "").strip() == model_name)
|
||||
if active_for_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit)
|
||||
raise ModelConcurrencyLimitError(model_name, limit, scope="model")
|
||||
|
||||
if user_model_concurrency_limit is not None:
|
||||
model_name, user_id, limit = user_model_concurrency_limit
|
||||
active_for_user_model = sum(
|
||||
1
|
||||
for r in self._runs.values()
|
||||
if r.status in (RunStatus.pending, RunStatus.running)
|
||||
and str((r.metadata or {}).get("model_name") or "").strip() == model_name
|
||||
and str((r.metadata or {}).get("user_id") or "").strip() == user_id
|
||||
)
|
||||
if active_for_user_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit, scope="user_model", user_id=user_id)
|
||||
|
||||
record = RunRecord(
|
||||
run_id=run_id,
|
||||
@@ -316,9 +329,11 @@ class ConflictError(Exception):
|
||||
class ModelConcurrencyLimitError(Exception):
|
||||
"""Raised when a model-specific in-flight run limit has been reached."""
|
||||
|
||||
def __init__(self, model_name: str, limit: int) -> None:
|
||||
def __init__(self, model_name: str, limit: int, *, scope: str = "model", user_id: str | None = None) -> None:
|
||||
self.model_name = model_name
|
||||
self.limit = limit
|
||||
self.scope = scope
|
||||
self.user_id = user_id
|
||||
super().__init__(f"Model {model_name} has reached its concurrency limit")
|
||||
|
||||
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Small in-memory gate used by legacy model-concurrency tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
|
||||
class ModelConcurrencyGate:
|
||||
"""Track per-model active counts with atomic try-acquire semantics."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = asyncio.Lock()
|
||||
self._counts: dict[str, int] = {}
|
||||
|
||||
async def try_acquire(self, model_name: str, limit: int) -> bool:
|
||||
key = str(model_name).strip()
|
||||
if not key:
|
||||
return True
|
||||
async with self._lock:
|
||||
current = self._counts.get(key, 0)
|
||||
if current >= max(1, int(limit)):
|
||||
return False
|
||||
self._counts[key] = current + 1
|
||||
return True
|
||||
|
||||
async def release(self, model_name: str) -> None:
|
||||
key = str(model_name).strip()
|
||||
if not key:
|
||||
return
|
||||
async with self._lock:
|
||||
current = self._counts.get(key, 0)
|
||||
if current <= 1:
|
||||
self._counts.pop(key, None)
|
||||
else:
|
||||
self._counts[key] = current - 1
|
||||
|
||||
async def snapshot(self) -> dict[str, int]:
|
||||
async with self._lock:
|
||||
return dict(self._counts)
|
||||
Reference in new issue
Block a user