fix: isolate user concurrency from internal agent runs

This commit is contained in:
吉鑫升 committed 2026-06-30 14:56:29 +08:00
1 parent 124eb04a6b
commit c23350f17b
8 files changed
+162 -10

No files matched your search

@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import os
import re
import time
import uuid
@@ -41,12 +42,68 @@ from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
RUN_SCOPE_EXTERNAL_ROOT = "external_root"
RUN_SCOPE_ORCHESTRATION_CHILD = "orchestration_child"
RUN_SCOPE_BACKGROUND_CHILD = "background_child"
_INTERNAL_RUN_SCOPES = {RUN_SCOPE_ORCHESTRATION_CHILD, RUN_SCOPE_BACKGROUND_CHILD}
_ALL_RUN_SCOPES = {RUN_SCOPE_EXTERNAL_ROOT, *_INTERNAL_RUN_SCOPES}
_INTERNAL_RUN_SCOPE_HEADER = "x-deerflow-internal-run-scope-token"
_INTERNAL_RUN_SCOPE_TOKEN = (
os.getenv("DEERFLOW_INTERNAL_RUN_SCOPE_TOKEN", "").strip()
or uuid.uuid4().hex
)
def internal_run_scope_headers() -> dict[str, str]:
"""Headers used by same-process loopback calls to mark internal child runs."""
return {_INTERNAL_RUN_SCOPE_HEADER: _INTERNAL_RUN_SCOPE_TOKEN}
def _metadata_run_scope(body: Any) -> str | None:
metadata = getattr(body, "metadata", None) or {}
if not isinstance(metadata, dict):
return None
raw = str(metadata.get("run_scope") or "").strip()
return raw or None
def _request_is_loopback(request: Request) -> bool:
client = getattr(request, "client", None)
host = str(getattr(client, "host", "") or "").strip()
return host in {"127.0.0.1", "::1", "localhost"}
def _has_internal_run_scope_header(request: Request) -> bool:
headers = getattr(request, "headers", None)
token = headers.get(_INTERNAL_RUN_SCOPE_HEADER) if headers is not None else None
return str(token or "") == _INTERNAL_RUN_SCOPE_TOKEN
def _resolve_run_scope(body: Any, request: Request, run_scope: str | None) -> str:
"""Return the trusted run scope used for concurrency accounting."""
if run_scope:
scope = str(run_scope).strip()
return scope if scope in _ALL_RUN_SCOPES else RUN_SCOPE_EXTERNAL_ROOT
scope = _metadata_run_scope(body)
if (
scope in _INTERNAL_RUN_SCOPES
and _request_is_loopback(request)
and _has_internal_run_scope_header(request)
):
return scope
if scope == RUN_SCOPE_EXTERNAL_ROOT:
return RUN_SCOPE_EXTERNAL_ROOT
return RUN_SCOPE_EXTERNAL_ROOT
async def _run_agent_with_concurrency_lease(
*,
concurrency_gate: Any | None,
lease_acquired: bool,
lease_metadata: dict[str, str],
lease_metadata: dict[str, Any],
heartbeat_interval_seconds: int,
run_agent_kwargs: dict[str, Any],
) -> None:
@@ -750,6 +807,7 @@ async def start_run(
request: Request,
*,
enforce_access: bool = True,
run_scope: str | None = None,
) -> RunRecord:
"""Create a RunRecord and launch the background agent task.
@@ -793,21 +851,29 @@ async def start_run(
effective_model_name = _resolve_effective_run_model_name(body)
effective_user_id = get_effective_user_id()
effective_run_scope = _resolve_run_scope(body, request, run_scope)
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
user_model_concurrency_limit = _get_user_model_concurrency_limit(effective_model_name)
user_model_concurrency_limit = (
_get_user_model_concurrency_limit(effective_model_name)
if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
else None
)
run_id = str(uuid.uuid4())
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
run_metadata["user_id"] = effective_user_id
run_metadata["run_scope"] = effective_run_scope
redis_lease_acquired = False
redis_gate_degraded = True
user_model_accounted = user_model_concurrency_limit is not None
lease_metadata = {
"run_id": run_id,
"thread_id": thread_id,
"model_name": effective_model_name,
"user_id": effective_user_id,
"user_model_accounted": user_model_accounted,
}
if concurrency_gate is not None:
acquire = await concurrency_gate.try_acquire(