fix: isolate user concurrency from internal agent runs
This commit is contained in:
1 parent
124eb04a6b
commit
c23350f17b
8 files changed
+162
-10
No files matched your search
+4
-1
@@ -14,7 +14,10 @@ class RedisConcurrencyConfig(BaseModel):
|
||||
default=None,
|
||||
description="Optional Redis password. You may also include credentials directly in url.",
|
||||
)
|
||||
namespace: str = Field(default="deerflow:concurrency", description="Redis key prefix for concurrency leases.")
|
||||
namespace: str = Field(
|
||||
default="cmzs_deerflow:concurrency",
|
||||
description="Redis key prefix for concurrency leases. Use a deployment/system prefix when Redis is shared.",
|
||||
)
|
||||
socket_connect_timeout_ms: int = Field(default=80, ge=1, le=10_000)
|
||||
socket_timeout_ms: int = Field(default=80, ge=1, le=10_000)
|
||||
acquire_timeout_ms: int = Field(default=120, ge=1, le=10_000)
|
||||
|
||||
+14
-4
@@ -110,7 +110,9 @@ end
|
||||
|
||||
redis.call('ZADD', KEYS[1], expires_at, run_id)
|
||||
redis.call('ZADD', KEYS[2], expires_at, run_id)
|
||||
redis.call('ZADD', KEYS[3], expires_at, run_id)
|
||||
if user_model_limit > 0 then
|
||||
redis.call('ZADD', KEYS[3], expires_at, run_id)
|
||||
end
|
||||
redis.call('ZADD', KEYS[4], expires_at, run_id)
|
||||
redis.call('SET', KEYS[5], payload, 'PX', ttl_ms)
|
||||
return {'acquired', ''}
|
||||
@@ -146,7 +148,7 @@ return {'acquired', ''}
|
||||
logger.debug("Failed to close Redis concurrency client", exc_info=True)
|
||||
|
||||
def _ns(self) -> str:
|
||||
return self.config.namespace.strip().rstrip(":") or "deerflow:concurrency"
|
||||
return self.config.namespace.strip().rstrip(":") or "cmzs_deerflow:concurrency"
|
||||
|
||||
def _keys(self, *, run_id: str, model_name: str, user_id: str, thread_id: str) -> list[str]:
|
||||
ns = self._ns()
|
||||
@@ -263,6 +265,7 @@ return {'acquired', ''}
|
||||
thread_id: str,
|
||||
model_name: str,
|
||||
user_id: str,
|
||||
user_model_accounted: bool = True,
|
||||
) -> None:
|
||||
if not self.enabled or not self._breaker.allow():
|
||||
return
|
||||
@@ -272,7 +275,10 @@ return {'acquired', ''}
|
||||
try:
|
||||
client = await self._redis()
|
||||
pipe = client.pipeline(transaction=False)
|
||||
for key in keys[:4]:
|
||||
refresh_keys = [keys[0], keys[1], keys[3]]
|
||||
if user_model_accounted:
|
||||
refresh_keys.append(keys[2])
|
||||
for key in refresh_keys:
|
||||
pipe.zadd(key, {run_id: expires_at})
|
||||
pipe.expire(keys[4], self.config.lease_ttl_seconds)
|
||||
await self._with_timeout(pipe.execute(), self.config.operation_timeout_ms)
|
||||
@@ -287,6 +293,7 @@ return {'acquired', ''}
|
||||
thread_id: str,
|
||||
model_name: str,
|
||||
user_id: str,
|
||||
user_model_accounted: bool = True,
|
||||
) -> None:
|
||||
if not self.enabled or not self._breaker.allow():
|
||||
return
|
||||
@@ -294,7 +301,10 @@ return {'acquired', ''}
|
||||
try:
|
||||
client = await self._redis()
|
||||
pipe = client.pipeline(transaction=False)
|
||||
for key in keys[:4]:
|
||||
release_keys = [keys[0], keys[1], keys[3]]
|
||||
if user_model_accounted:
|
||||
release_keys.append(keys[2])
|
||||
for key in release_keys:
|
||||
pipe.zrem(key, run_id)
|
||||
pipe.delete(keys[4])
|
||||
await self._with_timeout(pipe.execute(), self.config.operation_timeout_ms)
|
||||
|
||||
@@ -273,6 +273,7 @@ class RunManager:
|
||||
if r.status in (RunStatus.pending, RunStatus.running)
|
||||
and str((r.metadata or {}).get("model_name") or "").strip() == model_name
|
||||
and str((r.metadata or {}).get("user_id") or "").strip() == user_id
|
||||
and str((r.metadata or {}).get("run_scope") or "external_root").strip() == "external_root"
|
||||
)
|
||||
if active_for_user_model >= max(1, int(limit)):
|
||||
raise ModelConcurrencyLimitError(model_name, limit, scope="user_model", user_id=user_id)
|
||||
|
||||
Reference in new issue
Block a user