fix: isolate user concurrency from internal agent runs
This commit is contained in:
1 parent
124eb04a6b
commit
c23350f17b
8 files changed
+162
-10
No files matched your search
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
@@ -41,12 +42,68 @@ from deerflow.runtime.user_context import get_effective_user_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RUN_SCOPE_EXTERNAL_ROOT = "external_root"
|
||||
RUN_SCOPE_ORCHESTRATION_CHILD = "orchestration_child"
|
||||
RUN_SCOPE_BACKGROUND_CHILD = "background_child"
|
||||
_INTERNAL_RUN_SCOPES = {RUN_SCOPE_ORCHESTRATION_CHILD, RUN_SCOPE_BACKGROUND_CHILD}
|
||||
_ALL_RUN_SCOPES = {RUN_SCOPE_EXTERNAL_ROOT, *_INTERNAL_RUN_SCOPES}
|
||||
_INTERNAL_RUN_SCOPE_HEADER = "x-deerflow-internal-run-scope-token"
|
||||
_INTERNAL_RUN_SCOPE_TOKEN = (
|
||||
os.getenv("DEERFLOW_INTERNAL_RUN_SCOPE_TOKEN", "").strip()
|
||||
or uuid.uuid4().hex
|
||||
)
|
||||
|
||||
|
||||
def internal_run_scope_headers() -> dict[str, str]:
|
||||
"""Headers used by same-process loopback calls to mark internal child runs."""
|
||||
|
||||
return {_INTERNAL_RUN_SCOPE_HEADER: _INTERNAL_RUN_SCOPE_TOKEN}
|
||||
|
||||
|
||||
def _metadata_run_scope(body: Any) -> str | None:
|
||||
metadata = getattr(body, "metadata", None) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
return None
|
||||
raw = str(metadata.get("run_scope") or "").strip()
|
||||
return raw or None
|
||||
|
||||
|
||||
def _request_is_loopback(request: Request) -> bool:
|
||||
client = getattr(request, "client", None)
|
||||
host = str(getattr(client, "host", "") or "").strip()
|
||||
return host in {"127.0.0.1", "::1", "localhost"}
|
||||
|
||||
|
||||
def _has_internal_run_scope_header(request: Request) -> bool:
|
||||
headers = getattr(request, "headers", None)
|
||||
token = headers.get(_INTERNAL_RUN_SCOPE_HEADER) if headers is not None else None
|
||||
return str(token or "") == _INTERNAL_RUN_SCOPE_TOKEN
|
||||
|
||||
|
||||
def _resolve_run_scope(body: Any, request: Request, run_scope: str | None) -> str:
|
||||
"""Return the trusted run scope used for concurrency accounting."""
|
||||
|
||||
if run_scope:
|
||||
scope = str(run_scope).strip()
|
||||
return scope if scope in _ALL_RUN_SCOPES else RUN_SCOPE_EXTERNAL_ROOT
|
||||
|
||||
scope = _metadata_run_scope(body)
|
||||
if (
|
||||
scope in _INTERNAL_RUN_SCOPES
|
||||
and _request_is_loopback(request)
|
||||
and _has_internal_run_scope_header(request)
|
||||
):
|
||||
return scope
|
||||
if scope == RUN_SCOPE_EXTERNAL_ROOT:
|
||||
return RUN_SCOPE_EXTERNAL_ROOT
|
||||
return RUN_SCOPE_EXTERNAL_ROOT
|
||||
|
||||
|
||||
async def _run_agent_with_concurrency_lease(
|
||||
*,
|
||||
concurrency_gate: Any | None,
|
||||
lease_acquired: bool,
|
||||
lease_metadata: dict[str, str],
|
||||
lease_metadata: dict[str, Any],
|
||||
heartbeat_interval_seconds: int,
|
||||
run_agent_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
@@ -750,6 +807,7 @@ async def start_run(
|
||||
request: Request,
|
||||
*,
|
||||
enforce_access: bool = True,
|
||||
run_scope: str | None = None,
|
||||
) -> RunRecord:
|
||||
"""Create a RunRecord and launch the background agent task.
|
||||
|
||||
@@ -793,21 +851,29 @@ async def start_run(
|
||||
|
||||
effective_model_name = _resolve_effective_run_model_name(body)
|
||||
effective_user_id = get_effective_user_id()
|
||||
effective_run_scope = _resolve_run_scope(body, request, run_scope)
|
||||
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
|
||||
user_model_concurrency_limit = _get_user_model_concurrency_limit(effective_model_name)
|
||||
user_model_concurrency_limit = (
|
||||
_get_user_model_concurrency_limit(effective_model_name)
|
||||
if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
|
||||
else None
|
||||
)
|
||||
run_id = str(uuid.uuid4())
|
||||
|
||||
run_metadata = dict(body.metadata or {})
|
||||
run_metadata["model_name"] = effective_model_name
|
||||
run_metadata["user_id"] = effective_user_id
|
||||
run_metadata["run_scope"] = effective_run_scope
|
||||
|
||||
redis_lease_acquired = False
|
||||
redis_gate_degraded = True
|
||||
user_model_accounted = user_model_concurrency_limit is not None
|
||||
lease_metadata = {
|
||||
"run_id": run_id,
|
||||
"thread_id": thread_id,
|
||||
"model_name": effective_model_name,
|
||||
"user_id": effective_user_id,
|
||||
"user_model_accounted": user_model_accounted,
|
||||
}
|
||||
if concurrency_gate is not None:
|
||||
acquire = await concurrency_gate.try_acquire(
|
||||
|
||||
Reference in new issue
Block a user