并发版
This commit is contained in:
1 parent
cd2ca406a5
commit
d39b58255d
11 files changed
+530
-73
No files matched your search
@@ -218,6 +218,7 @@ class RunManager:
|
||||
kwargs: dict | None = None,
|
||||
multitask_strategy: str = "reject",
|
||||
model_concurrency_limit: tuple[str, int] | None = None,
|
||||
user_model_concurrency_limit: tuple[str, str, int] | None = None,
|
||||
) -> RunRecord:
|
||||
"""Atomically check for inflight runs and create a new one.
|
||||
|
||||
@@ -261,7 +262,19 @@ class RunManager:
|
||||
model_name, limit = model_concurrency_limit
|
||||
active_for_model = sum(1 for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running) and str((r.metadata or {}).get("model_name") or "").strip() == model_name)
|
||||
if active_for_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit)
|
||||
raise ModelConcurrencyLimitError(model_name, limit, scope="model")
|
||||
|
||||
if user_model_concurrency_limit is not None:
|
||||
model_name, user_id, limit = user_model_concurrency_limit
|
||||
active_for_user_model = sum(
|
||||
1
|
||||
for r in self._runs.values()
|
||||
if r.status in (RunStatus.pending, RunStatus.running)
|
||||
and str((r.metadata or {}).get("model_name") or "").strip() == model_name
|
||||
and str((r.metadata or {}).get("user_id") or "").strip() == user_id
|
||||
)
|
||||
if active_for_user_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit, scope="user_model", user_id=user_id)
|
||||
|
||||
record = RunRecord(
|
||||
run_id=run_id,
|
||||
@@ -316,9 +329,11 @@ class ConflictError(Exception):
|
||||
class ModelConcurrencyLimitError(Exception):
|
||||
"""Raised when a model-specific in-flight run limit has been reached."""
|
||||
|
||||
def __init__(self, model_name: str, limit: int) -> None:
|
||||
def __init__(self, model_name: str, limit: int, *, scope: str = "model", user_id: str | None = None) -> None:
|
||||
self.model_name = model_name
|
||||
self.limit = limit
|
||||
self.scope = scope
|
||||
self.user_id = user_id
|
||||
super().__init__(f"Model {model_name} has reached its concurrency limit")
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user