Merge branch 'master-dev' into 'master'

This commit is contained in:
吉鑫升 committed 2026-07-21 14:47:46 +08:00
commit 5436719bfb
39 files changed
+2395 -291

No files matched your search

@@ -853,8 +853,9 @@ async def _concurrency_sampler_loop(app: FastAPI) -> None:
logger.info("Concurrency sampler disabled by CONCURRENCY_SAMPLE_ENABLED")
return
store = getattr(app.state, "concurrency_sample_store", None)
run_mgr = getattr(app.state, "run_manager", None)
store = getattr(app.state, "concurrency_sample_store", None)
run_mgr = getattr(app.state, "run_manager", None)
concurrency_gate = getattr(app.state, "concurrency_gate", None)
if store is None or run_mgr is None:
logger.info("Concurrency sampler not started (no store / run manager — memory backend?)")
return
@@ -875,11 +876,16 @@ async def _concurrency_sampler_loop(app: FastAPI) -> None:
try:
# Runtime toggle (admin can turn sampling off without a restart).
# mtime-cached read, so this is cheap every tick.
from deerflow.config.system_settings import get_concurrency_monitor_settings
if get_concurrency_monitor_settings().enabled:
active = await run_mgr.list_active()
await store.record(len(active))
from deerflow.config.system_settings import get_concurrency_monitor_settings
if get_concurrency_monitor_settings().enabled:
active_count = None
if concurrency_gate is not None:
active_count = await concurrency_gate.count_active()
if active_count is None:
active = await run_mgr.list_active()
active_count = len(active)
await store.record(active_count)
ticks += 1
if ticks % purge_every_ticks == 0:
from datetime import UTC, datetime, timedelta
@@ -367,8 +367,11 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
run_events_config = getattr(config, "run_events", None)
app.state.run_event_store = make_run_event_store(run_events_config)
# RunManager with store backing for persistence
app.state.run_manager = RunManager(store=app.state.run_store)
# RunManager with store backing for persistence
app.state.run_manager = RunManager(store=app.state.run_store)
from deerflow.runtime import make_redis_concurrency_gate
app.state.concurrency_gate = make_redis_concurrency_gate(config)
try:
yield
@@ -382,7 +385,10 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
await queue.stop()
set_auto_ingest_queue(None)
await close_engine()
await close_engine()
gate = getattr(app.state, "concurrency_gate", None)
if gate is not None:
await gate.aclose()
# ---------------------------------------------------------------------------
@@ -408,7 +414,12 @@ get_run_manager: Callable[[Request], RunManager] = _require("run_manager", "Run
get_checkpointer: Callable[[Request], Checkpointer] = _require("checkpointer", "Checkpointer")
get_run_event_store: Callable[[Request], RunEventStore] = _require("run_event_store", "Run event store")
get_feedback_repo: Callable[[Request], FeedbackRepository] = _require("feedback_repo", "Feedback")
get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store")
get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store")
def get_concurrency_gate(request: Request):
"""Return the optional cross-worker concurrency gate."""
return getattr(request.app.state, "concurrency_gate", None)
def get_store(request: Request):
@@ -32,6 +32,7 @@ from app.gateway.roundtable_model_fallback import (
)
from app.gateway.roundtable_run_policy import roundtable_run_policy
from app.gateway.roundtable_seat_skills import append_seat_skill_directive
from app.gateway.services import RUN_SCOPE_BACKGROUND_CHILD
from app.gateway.routers._roundtable_seed import (
COORDINATOR_AGENT_ID,
DASHBOARD_AGENT_ID,
@@ -346,6 +347,7 @@ async def _run_agent_turn(
body = RunCreateRequest(
assistant_id="lead_agent",
input={"messages": [{"type": "human", "content": [{"type": "text", "text": message}], "additional_kwargs": {}}]},
metadata={"run_scope": RUN_SCOPE_BACKGROUND_CHILD},
config={"recursion_limit": 1000},
context={
"agent_name": agent_name,
@@ -374,7 +376,12 @@ async def _run_agent_turn(
# 落 checkpoint)。额度满时本轮在此 await 排队,等同用户某个在跑轮次释放额度后再开。
async with _user_background_semaphore(user_id):
try:
record = await start_run(body, thread_id, request) # type: ignore[arg-type]
record = await start_run( # type: ignore[arg-type]
body,
thread_id,
request,
run_scope=RUN_SCOPE_BACKGROUND_CHILD,
)
except Exception as exc: # noqa: BLE001 — start_run 报错(409 抢占 / DB 抖动等)也要落诊断
if diag is not None:
await diag.record(
@@ -22,7 +22,7 @@ from fastapi import APIRouter, Query, Request
from pydantic import BaseModel
from sqlalchemy import select
from app.gateway.deps import get_concurrency_sample_store, get_run_manager
from app.gateway.deps import get_concurrency_gate, get_concurrency_sample_store, get_run_manager
from app.gateway.routers.admin_users import _require_admin
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.persistence.agents.model import AgentRow
@@ -149,6 +149,15 @@ def _elapsed_seconds(created_at: str, now: datetime) -> int:
return max(0, int(delta))
def _elapsed_seconds_from_epoch(created_at: float, now: datetime) -> int:
try:
if created_at <= 0:
return 0
return max(0, int(now.timestamp() - float(created_at)))
except Exception:
return 0
def _kind_for(metadata: dict | None) -> str:
"""Coarse human label for the run's origin, from its thread metadata."""
meta = metadata or {}
@@ -218,14 +227,46 @@ async def list_active_runs(request: Request) -> ActiveRunsResponse:
"""List every in-flight run with who/what is driving it (admin only)."""
await _require_admin(request)
run_mgr = get_run_manager(request)
records = await run_mgr.list_active()
gate = get_concurrency_gate(request)
leases = await gate.list_active() if gate is not None else None
records = [] if leases is not None else await run_mgr.list_active()
now = datetime.now(UTC)
maps = await _enrich(records)
enrich_source = leases if leases is not None else records
maps = await _enrich(enrich_source)
thread_meta = maps["thread_meta"]
user_emails = maps["user_emails"]
agent_names = maps["agent_names"]
items: list[ActiveRunItem] = []
if leases is not None:
for r in leases:
meta_row = thread_meta.get(r.thread_id)
user_id = r.user_id or (meta_row.user_id if meta_row else "") or ""
metadata = (meta_row.metadata_json if meta_row else None) or {}
assistant_id = r.assistant_id or ""
if assistant_id in _DEFAULT_ASSISTANT_IDS:
agent_name = "榛樿鍔╂墜"
else:
agent_name = agent_names.get(assistant_id) or assistant_id
items.append(
ActiveRunItem(
run_id=r.run_id,
thread_id=r.thread_id,
status="running",
created_at=datetime.fromtimestamp(r.created_at, tz=UTC).isoformat() if r.created_at else "",
elapsed_seconds=_elapsed_seconds_from_epoch(r.created_at, now),
user_id=user_id,
user_name=user_emails.get(user_id) or user_id or "鏈煡鐢ㄦ埛",
agent_id=assistant_id,
agent_name=agent_name,
model_name=r.model_name,
thread_title=(meta_row.display_name if meta_row else "") or "",
kind=_kind_for(metadata),
multitask_strategy="reject",
)
)
return ActiveRunsResponse(count=len(items), server_time=now.isoformat(), runs=items)
for r in records:
meta_row = thread_meta.get(r.thread_id)
user_id = (meta_row.user_id if meta_row else "") or ""
@@ -65,6 +65,7 @@ from langgraph.checkpoint.base import empty_checkpoint
from app.gateway.roundtable_diag import record_foreground_diag
from app.gateway.roundtable_model_fallback import fallback_model_chain, is_llm_error_message, reason_text
from app.gateway.roundtable_run_policy import merge_skill_stop_names, roundtable_run_policy
from app.gateway.services import RUN_SCOPE_ORCHESTRATION_CHILD, internal_run_scope_headers
from deerflow.agents.roundtable_orchestrator.extraction_spec import build_summary_directive
from app.gateway.roundtable_seat_skills import append_seat_skill_directive
@@ -325,7 +326,11 @@ def _auth_headers(request: Request) -> dict[str, str]:
token = request.headers.get("authorization")
if not token:
raise HTTPException(status_code=401, detail="Missing Authorization header")
return {"Authorization": token, "Content-Type": "application/json"}
return {
"Authorization": token,
"Content-Type": "application/json",
**internal_run_scope_headers(),
}
async def _create_thread(
@@ -694,6 +699,7 @@ def _run_payload(
}
]
},
"metadata": {"run_scope": RUN_SCOPE_ORCHESTRATION_CHILD},
"config": {"recursion_limit": 1000},
"context": {
"agent_name": agent_name,
@@ -24,7 +24,6 @@ from deerflow.config.system_settings import (
PublishSquare,
PublishSquaresSettings,
SkillCompressionSettings,
SystemSettings,
get_module_tag_ids,
list_publish_squares,
load_system_settings,
@@ -226,12 +225,16 @@ class ModelConcurrencyResponse(BaseModel):
enabled: bool = Field(..., description="Whether per-model concurrency limiting is active")
default_limit: int = Field(..., description="Fallback max concurrent runs for models without an explicit override")
model_limits: dict[str, int] = Field(default_factory=dict, description="Per-model overrides: model_name -> max concurrent runs")
user_default_limit: int = Field(..., description="Fallback max concurrent runs per user and model. 0 disables the per-user fallback")
user_model_limits: dict[str, int] = Field(default_factory=dict, description="Per-user per-model overrides: model_name -> max concurrent runs for one user")
class ModelConcurrencyUpdateRequest(BaseModel):
enabled: bool | None = None
default_limit: int | None = Field(default=None, ge=1, le=10_000)
model_limits: dict[str, int] | None = None
user_default_limit: int | None = Field(default=None, ge=0, le=10_000)
user_model_limits: dict[str, int] | None = None
def _skill_compression_to_response(settings: SkillCompressionSettings) -> SkillCompressionResponse:
@@ -255,10 +258,24 @@ def _model_concurrency_to_response(settings: ModelConcurrencySettings) -> ModelC
if normalized_limit < 1:
continue
cleaned[model_name] = min(normalized_limit, 10_000)
cleaned_user: dict[str, int] = {}
for name, limit in (settings.user_model_limits or {}).items():
model_name = str(name).strip()
if not model_name:
continue
try:
normalized_limit = int(limit)
except (TypeError, ValueError):
continue
if normalized_limit < 1:
continue
cleaned_user[model_name] = min(normalized_limit, 10_000)
return ModelConcurrencyResponse(
enabled=settings.enabled,
default_limit=max(1, min(int(settings.default_limit), 10_000)),
model_limits=cleaned,
user_default_limit=max(0, min(int(getattr(settings, "user_default_limit", 0) or 0), 10_000)),
user_model_limits=cleaned_user,
)
@@ -336,6 +353,22 @@ async def update_model_concurrency(
continue
cleaned[model_name] = min(normalized_limit, 10_000)
update["model_limits"] = cleaned
if "user_default_limit" in fields and fields["user_default_limit"] is not None:
update["user_default_limit"] = max(0, min(int(fields["user_default_limit"]), 10_000))
if "user_model_limits" in fields and fields["user_model_limits"] is not None:
cleaned_user: dict[str, int] = {}
for name, limit in (body.user_model_limits or {}).items():
model_name = str(name).strip()
if not model_name:
continue
try:
normalized_limit = int(limit)
except (TypeError, ValueError):
continue
if normalized_limit < 1:
continue
cleaned_user[model_name] = min(normalized_limit, 10_000)
update["user_model_limits"] = cleaned_user
settings = current.model_concurrency.model_copy(update=update)
saved = save_system_settings(current.model_copy(update={"model_concurrency": settings}))
return _model_concurrency_to_response(saved.model_concurrency)
@@ -10,15 +10,17 @@ from __future__ import annotations
import asyncio
import json
import logging
import os
import re
import time
import uuid
from collections.abc import Mapping
from typing import Any
from fastapi import HTTPException, Request
from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.deps import get_concurrency_gate, get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config, validate_agent_id
@@ -36,9 +38,105 @@ from deerflow.runtime import (
UnsupportedStrategyError,
run_agent,
)
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
RUN_SCOPE_EXTERNAL_ROOT = "external_root"
RUN_SCOPE_ORCHESTRATION_CHILD = "orchestration_child"
RUN_SCOPE_BACKGROUND_CHILD = "background_child"
_INTERNAL_RUN_SCOPES = {RUN_SCOPE_ORCHESTRATION_CHILD, RUN_SCOPE_BACKGROUND_CHILD}
_ALL_RUN_SCOPES = {RUN_SCOPE_EXTERNAL_ROOT, *_INTERNAL_RUN_SCOPES}
_INTERNAL_RUN_SCOPE_HEADER = "x-deerflow-internal-run-scope-token"
_INTERNAL_RUN_SCOPE_TOKEN = (
os.getenv("DEERFLOW_INTERNAL_RUN_SCOPE_TOKEN", "").strip()
or uuid.uuid4().hex
)
def internal_run_scope_headers() -> dict[str, str]:
"""Headers used by same-process loopback calls to mark internal child runs."""
return {_INTERNAL_RUN_SCOPE_HEADER: _INTERNAL_RUN_SCOPE_TOKEN}
def _metadata_run_scope(body: Any) -> str | None:
metadata = getattr(body, "metadata", None) or {}
if not isinstance(metadata, dict):
return None
raw = str(metadata.get("run_scope") or "").strip()
return raw or None
def _request_is_loopback(request: Request) -> bool:
client = getattr(request, "client", None)
host = str(getattr(client, "host", "") or "").strip()
return host in {"127.0.0.1", "::1", "localhost"}
def _has_internal_run_scope_header(request: Request) -> bool:
headers = getattr(request, "headers", None)
token = headers.get(_INTERNAL_RUN_SCOPE_HEADER) if headers is not None else None
return str(token or "") == _INTERNAL_RUN_SCOPE_TOKEN
def _resolve_run_scope(body: Any, request: Request, run_scope: str | None) -> str:
"""Return the trusted run scope used for concurrency accounting."""
if run_scope:
scope = str(run_scope).strip()
return scope if scope in _ALL_RUN_SCOPES else RUN_SCOPE_EXTERNAL_ROOT
scope = _metadata_run_scope(body)
if (
scope in _INTERNAL_RUN_SCOPES
and _request_is_loopback(request)
and _has_internal_run_scope_header(request)
):
return scope
if scope == RUN_SCOPE_EXTERNAL_ROOT:
return RUN_SCOPE_EXTERNAL_ROOT
return RUN_SCOPE_EXTERNAL_ROOT
async def _run_agent_with_concurrency_lease(
*,
concurrency_gate: Any | None,
lease_acquired: bool,
lease_metadata: dict[str, Any],
heartbeat_interval_seconds: int,
run_agent_kwargs: dict[str, Any],
) -> None:
"""Run the agent while refreshing and finally releasing an optional lease."""
heartbeat_task: asyncio.Task | None = None
async def _heartbeat_loop() -> None:
if concurrency_gate is None or not lease_acquired:
return
interval = max(5, int(heartbeat_interval_seconds or 20))
while True:
await asyncio.sleep(interval)
await concurrency_gate.heartbeat(**lease_metadata)
if concurrency_gate is not None and lease_acquired:
heartbeat_task = asyncio.create_task(
_heartbeat_loop(),
name=f"concurrency-heartbeat:{lease_metadata.get('run_id')}",
)
try:
await run_agent(**run_agent_kwargs)
finally:
if heartbeat_task is not None:
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
if concurrency_gate is not None and lease_acquired:
await concurrency_gate.release(**lease_metadata)
# ---------------------------------------------------------------------------
# SSE formatting
@@ -498,6 +596,49 @@ def _resolve_effective_run_model_name(body: Any) -> str:
return default_model_name
def _resolve_effective_model_name(body: Any, app_config: Any | None = None) -> str:
"""Compatibility wrapper for resolving the effective run model."""
if app_config is None:
return _resolve_effective_run_model_name(body)
default_model_name = app_config.models[0].name if app_config.models else None
if not default_model_name:
raise ValueError("No chat models are configured. Please configure at least one model in config.yaml.")
requested_model_name = _extract_requested_model_name(body)
if requested_model_name and app_config.get_model_config(requested_model_name):
return requested_model_name
agent_id = _extract_effective_agent_id(body)
if agent_id:
try:
agent_config = load_agent_config(agent_id)
except Exception:
agent_config = None
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
if agent_model_name and app_config.get_model_config(agent_model_name):
return agent_model_name
return default_model_name
def _resolve_model_concurrency_limit(model_name: str, app_config: Any) -> int | None:
"""Read legacy per-model max-concurrency values from model config extras."""
model_config = app_config.get_model_config(model_name)
extras = getattr(model_config, "model_extra", None) or {}
for key in ("max_concurrency", "model_concurrency_limit"):
raw = extras.get(key)
if raw is None:
continue
try:
limit = int(raw)
except (TypeError, ValueError):
continue
if limit > 0:
return limit
return None
def _get_model_concurrency_limit(model_name: str) -> int | None:
"""Return the configured limit for a model, or ``None`` when limiting is off."""
settings = load_system_settings().model_concurrency
@@ -512,6 +653,25 @@ def _get_model_concurrency_limit(model_name: str) -> int | None:
return max(1, limit)
def _get_user_model_concurrency_limit(model_name: str) -> int | None:
"""Return the configured per-user limit for a model, or ``None`` when off."""
settings = load_system_settings().model_concurrency
if not settings.enabled:
return None
raw_limit = settings.user_model_limits.get(
model_name,
getattr(settings, "user_default_limit", 0) or 0,
)
try:
limit = int(raw_limit)
except (TypeError, ValueError):
limit = int(getattr(settings, "user_default_limit", 0) or 0)
if limit < 1:
return None
return max(1, limit)
async def _sync_legacy_agent_records(store) -> None:
"""Import existing folder-backed agents as built-in records.
@@ -647,6 +807,7 @@ async def start_run(
request: Request,
*,
enforce_access: bool = True,
run_scope: str | None = None,
) -> RunRecord:
"""Create a RunRecord and launch the background agent task.
@@ -674,6 +835,7 @@ async def start_run(
bridge = get_stream_bridge(request)
run_mgr = get_run_manager(request)
run_ctx = get_run_context(request)
concurrency_gate = get_concurrency_gate(request)
disconnect = DisconnectMode.cancel if body.on_disconnect == "cancel" else DisconnectMode.continue_
@@ -688,10 +850,56 @@ async def start_run(
t_post_enforce = time.perf_counter()
effective_model_name = _resolve_effective_run_model_name(body)
effective_user_id = get_effective_user_id()
effective_run_scope = _resolve_run_scope(body, request, run_scope)
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
user_model_concurrency_limit = (
_get_user_model_concurrency_limit(effective_model_name)
if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
else None
)
run_id = str(uuid.uuid4())
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
run_metadata["user_id"] = effective_user_id
run_metadata["run_scope"] = effective_run_scope
redis_lease_acquired = False
redis_gate_degraded = True
user_model_accounted = user_model_concurrency_limit is not None
lease_metadata = {
"run_id": run_id,
"thread_id": thread_id,
"model_name": effective_model_name,
"user_id": effective_user_id,
"user_model_accounted": user_model_accounted,
}
if concurrency_gate is not None:
acquire = await concurrency_gate.try_acquire(
run_id=run_id,
thread_id=thread_id,
model_name=effective_model_name,
user_id=effective_user_id,
assistant_id=body.assistant_id or "",
model_limit=model_concurrency_limit,
user_model_limit=user_model_concurrency_limit,
reject_thread=body.multitask_strategy == "reject",
)
redis_lease_acquired = acquire.acquired
redis_gate_degraded = acquire.degraded
if acquire.rejected:
if acquire.reason == "thread":
raise HTTPException(
status_code=409,
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
)
if acquire.reason == "user_model":
raise HTTPException(
status_code=429,
detail="This account has reached the concurrency limit for the selected model. Please try again later.",
)
raise HTTPException(status_code=429, detail="The selected model is busy. Please try again later.")
try:
record = await run_mgr.create_or_reject(
@@ -701,19 +909,35 @@ async def start_run(
metadata=run_metadata,
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if model_concurrency_limit is not None else None,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if redis_gate_degraded and model_concurrency_limit is not None else None,
user_model_concurrency_limit=(effective_model_name, effective_user_id, user_model_concurrency_limit) if redis_gate_degraded and user_model_concurrency_limit is not None else None,
run_id=run_id,
)
except ModelConcurrencyLimitError as exc:
raise HTTPException(status_code=429, detail="当前使用人数较多,请稍等一下再试。") from exc
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
if getattr(exc, "scope", "") == "user_model":
detail = "This account has reached the concurrency limit for the selected model. Please try again later."
else:
detail = "The selected model is busy. Please try again later."
raise HTTPException(status_code=429, detail=detail) from exc
except ConflictError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
raise HTTPException(
status_code=409,
detail="当前对话还在生成回答,请等待本轮回复结束,或先点「停止」再发送新的问题。",
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
) from exc
except UnsupportedStrategyError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise HTTPException(status_code=501, detail=str(exc)) from exc
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
t_post_create = time.perf_counter()
# Upsert thread metadata so the thread appears in /threads/search,
@@ -751,7 +975,12 @@ async def start_run(
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
t_post_thread_upsert = time.perf_counter()
agent_factory = resolve_agent_factory(body.assistant_id)
try:
agent_factory = resolve_agent_factory(body.assistant_id)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
graph_input = normalize_input(body.input)
# Admin-configurable prompt prefix: when the caller opted in via
@@ -789,24 +1018,38 @@ async def start_run(
stream_modes = normalize_stream_modes(body.stream_mode)
t_pre_task = time.perf_counter()
task = asyncio.create_task(
run_agent(
bridge,
run_mgr,
record,
ctx=run_ctx,
agent_factory=agent_factory,
graph_input=graph_input,
config=config,
stream_modes=stream_modes,
stream_subgraphs=body.stream_subgraphs,
interrupt_before=body.interrupt_before,
interrupt_after=body.interrupt_after,
# 透传 SDK ``Command``:resume / update / goto。Pregel 会据此从最近
# checkpoint 处恢复,而不是把 graph_input 当新一轮输入跑。
command=getattr(body, "command", None),
heartbeat_interval_seconds = 20
if concurrency_gate is not None:
heartbeat_interval_seconds = int(
getattr(getattr(concurrency_gate, "config", None), "heartbeat_interval_seconds", 20) or 20
)
)
try:
task = asyncio.create_task(
_run_agent_with_concurrency_lease(
concurrency_gate=concurrency_gate,
lease_acquired=redis_lease_acquired,
lease_metadata=lease_metadata,
heartbeat_interval_seconds=heartbeat_interval_seconds,
run_agent_kwargs={
"bridge": bridge,
"run_manager": run_mgr,
"record": record,
"ctx": run_ctx,
"agent_factory": agent_factory,
"graph_input": graph_input,
"config": config,
"stream_modes": stream_modes,
"stream_subgraphs": body.stream_subgraphs,
"interrupt_before": body.interrupt_before,
"interrupt_after": body.interrupt_after,
"command": getattr(body, "command", None),
},
)
)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
record.task = task
# Title sync is handled by worker.py's finally block which reads the