# Conflicts:
#	frontend-web/src/components/workspace/workspace-nav-menu.tsx
#	offline-backend-20260512/backend/app/gateway/routers/multi_agent.py
This commit is contained in:
fengxiangdebaozi committed 2026-07-26 20:55:50 +08:00
commit 239b798c5d
42 files changed
+2416 -282

No files matched your search

@@ -10,15 +10,17 @@ from __future__ import annotations
import asyncio
import json
import logging
import os
import re
import time
import uuid
from collections.abc import Mapping
from typing import Any
from fastapi import HTTPException, Request
from langchain_core.messages import HumanMessage
from app.gateway.deps import get_run_context, get_run_manager, get_stream_bridge
from app.gateway.deps import get_concurrency_gate, get_run_context, get_run_manager, get_stream_bridge
from app.gateway.utils import sanitize_log_param
from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config, validate_agent_id
@@ -36,9 +38,105 @@ from deerflow.runtime import (
UnsupportedStrategyError,
run_agent,
)
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
RUN_SCOPE_EXTERNAL_ROOT = "external_root"
RUN_SCOPE_ORCHESTRATION_CHILD = "orchestration_child"
RUN_SCOPE_BACKGROUND_CHILD = "background_child"
_INTERNAL_RUN_SCOPES = {RUN_SCOPE_ORCHESTRATION_CHILD, RUN_SCOPE_BACKGROUND_CHILD}
_ALL_RUN_SCOPES = {RUN_SCOPE_EXTERNAL_ROOT, *_INTERNAL_RUN_SCOPES}
_INTERNAL_RUN_SCOPE_HEADER = "x-deerflow-internal-run-scope-token"
_INTERNAL_RUN_SCOPE_TOKEN = (
os.getenv("DEERFLOW_INTERNAL_RUN_SCOPE_TOKEN", "").strip()
or uuid.uuid4().hex
)
def internal_run_scope_headers() -> dict[str, str]:
"""Headers used by same-process loopback calls to mark internal child runs."""
return {_INTERNAL_RUN_SCOPE_HEADER: _INTERNAL_RUN_SCOPE_TOKEN}
def _metadata_run_scope(body: Any) -> str | None:
metadata = getattr(body, "metadata", None) or {}
if not isinstance(metadata, dict):
return None
raw = str(metadata.get("run_scope") or "").strip()
return raw or None
def _request_is_loopback(request: Request) -> bool:
client = getattr(request, "client", None)
host = str(getattr(client, "host", "") or "").strip()
return host in {"127.0.0.1", "::1", "localhost"}
def _has_internal_run_scope_header(request: Request) -> bool:
headers = getattr(request, "headers", None)
token = headers.get(_INTERNAL_RUN_SCOPE_HEADER) if headers is not None else None
return str(token or "") == _INTERNAL_RUN_SCOPE_TOKEN
def _resolve_run_scope(body: Any, request: Request, run_scope: str | None) -> str:
"""Return the trusted run scope used for concurrency accounting."""
if run_scope:
scope = str(run_scope).strip()
return scope if scope in _ALL_RUN_SCOPES else RUN_SCOPE_EXTERNAL_ROOT
scope = _metadata_run_scope(body)
if (
scope in _INTERNAL_RUN_SCOPES
and _request_is_loopback(request)
and _has_internal_run_scope_header(request)
):
return scope
if scope == RUN_SCOPE_EXTERNAL_ROOT:
return RUN_SCOPE_EXTERNAL_ROOT
return RUN_SCOPE_EXTERNAL_ROOT
async def _run_agent_with_concurrency_lease(
*,
concurrency_gate: Any | None,
lease_acquired: bool,
lease_metadata: dict[str, Any],
heartbeat_interval_seconds: int,
run_agent_kwargs: dict[str, Any],
) -> None:
"""Run the agent while refreshing and finally releasing an optional lease."""
heartbeat_task: asyncio.Task | None = None
async def _heartbeat_loop() -> None:
if concurrency_gate is None or not lease_acquired:
return
interval = max(5, int(heartbeat_interval_seconds or 20))
while True:
await asyncio.sleep(interval)
await concurrency_gate.heartbeat(**lease_metadata)
if concurrency_gate is not None and lease_acquired:
heartbeat_task = asyncio.create_task(
_heartbeat_loop(),
name=f"concurrency-heartbeat:{lease_metadata.get('run_id')}",
)
try:
await run_agent(**run_agent_kwargs)
finally:
if heartbeat_task is not None:
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
if concurrency_gate is not None and lease_acquired:
await concurrency_gate.release(**lease_metadata)
# ---------------------------------------------------------------------------
# SSE formatting
@@ -506,6 +604,49 @@ def _resolve_effective_run_model_name(body: Any) -> str:
return default_model_name
def _resolve_effective_model_name(body: Any, app_config: Any | None = None) -> str:
"""Compatibility wrapper for resolving the effective run model."""
if app_config is None:
return _resolve_effective_run_model_name(body)
default_model_name = app_config.models[0].name if app_config.models else None
if not default_model_name:
raise ValueError("No chat models are configured. Please configure at least one model in config.yaml.")
requested_model_name = _extract_requested_model_name(body)
if requested_model_name and app_config.get_model_config(requested_model_name):
return requested_model_name
agent_id = _extract_effective_agent_id(body)
if agent_id:
try:
agent_config = load_agent_config(agent_id)
except Exception:
agent_config = None
agent_model_name = str(getattr(agent_config, "model", "") or "").strip()
if agent_model_name and app_config.get_model_config(agent_model_name):
return agent_model_name
return default_model_name
def _resolve_model_concurrency_limit(model_name: str, app_config: Any) -> int | None:
"""Read legacy per-model max-concurrency values from model config extras."""
model_config = app_config.get_model_config(model_name)
extras = getattr(model_config, "model_extra", None) or {}
for key in ("max_concurrency", "model_concurrency_limit"):
raw = extras.get(key)
if raw is None:
continue
try:
limit = int(raw)
except (TypeError, ValueError):
continue
if limit > 0:
return limit
return None
def _get_model_concurrency_limit(model_name: str) -> int | None:
"""Return the configured limit for a model, or ``None`` when limiting is off."""
settings = load_system_settings().model_concurrency
@@ -520,6 +661,25 @@ def _get_model_concurrency_limit(model_name: str) -> int | None:
return max(1, limit)
def _get_user_model_concurrency_limit(model_name: str) -> int | None:
"""Return the configured per-user limit for a model, or ``None`` when off."""
settings = load_system_settings().model_concurrency
if not settings.enabled:
return None
raw_limit = settings.user_model_limits.get(
model_name,
getattr(settings, "user_default_limit", 0) or 0,
)
try:
limit = int(raw_limit)
except (TypeError, ValueError):
limit = int(getattr(settings, "user_default_limit", 0) or 0)
if limit < 1:
return None
return max(1, limit)
async def _sync_legacy_agent_records(store) -> None:
"""Import existing folder-backed agents as built-in records.
@@ -655,6 +815,7 @@ async def start_run(
request: Request,
*,
enforce_access: bool = True,
run_scope: str | None = None,
) -> RunRecord:
"""Create a RunRecord and launch the background agent task.
@@ -682,6 +843,7 @@ async def start_run(
bridge = get_stream_bridge(request)
run_mgr = get_run_manager(request)
run_ctx = get_run_context(request)
concurrency_gate = get_concurrency_gate(request)
disconnect = DisconnectMode.cancel if body.on_disconnect == "cancel" else DisconnectMode.continue_
@@ -696,10 +858,56 @@ async def start_run(
t_post_enforce = time.perf_counter()
effective_model_name = _resolve_effective_run_model_name(body)
effective_user_id = get_effective_user_id()
effective_run_scope = _resolve_run_scope(body, request, run_scope)
model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
user_model_concurrency_limit = (
_get_user_model_concurrency_limit(effective_model_name)
if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
else None
)
run_id = str(uuid.uuid4())
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
run_metadata["user_id"] = effective_user_id
run_metadata["run_scope"] = effective_run_scope
redis_lease_acquired = False
redis_gate_degraded = True
user_model_accounted = user_model_concurrency_limit is not None
lease_metadata = {
"run_id": run_id,
"thread_id": thread_id,
"model_name": effective_model_name,
"user_id": effective_user_id,
"user_model_accounted": user_model_accounted,
}
if concurrency_gate is not None:
acquire = await concurrency_gate.try_acquire(
run_id=run_id,
thread_id=thread_id,
model_name=effective_model_name,
user_id=effective_user_id,
assistant_id=body.assistant_id or "",
model_limit=model_concurrency_limit,
user_model_limit=user_model_concurrency_limit,
reject_thread=body.multitask_strategy == "reject",
)
redis_lease_acquired = acquire.acquired
redis_gate_degraded = acquire.degraded
if acquire.rejected:
if acquire.reason == "thread":
raise HTTPException(
status_code=409,
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
)
if acquire.reason == "user_model":
raise HTTPException(
status_code=429,
detail="This account has reached the concurrency limit for the selected model. Please try again later.",
)
raise HTTPException(status_code=429, detail="The selected model is busy. Please try again later.")
try:
record = await run_mgr.create_or_reject(
@@ -709,19 +917,35 @@ async def start_run(
metadata=run_metadata,
kwargs={"input": body.input, "config": body.config},
multitask_strategy=body.multitask_strategy,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if model_concurrency_limit is not None else None,
model_concurrency_limit=(effective_model_name, model_concurrency_limit) if redis_gate_degraded and model_concurrency_limit is not None else None,
user_model_concurrency_limit=(effective_model_name, effective_user_id, user_model_concurrency_limit) if redis_gate_degraded and user_model_concurrency_limit is not None else None,
run_id=run_id,
)
except ModelConcurrencyLimitError as exc:
raise HTTPException(status_code=429, detail="当前使用人数较多,请稍等一下再试。") from exc
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
if getattr(exc, "scope", "") == "user_model":
detail = "This account has reached the concurrency limit for the selected model. Please try again later."
else:
detail = "The selected model is busy. Please try again later."
raise HTTPException(status_code=429, detail=detail) from exc
except ConflictError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
# 同一会话上一轮回答还没结束,又提交了新问题 → 拒绝。给用户中文提示,
# 而不是把内部英文「Thread ... already has an active run」直接抛到前端。
raise HTTPException(
status_code=409,
detail="当前对话还在生成回答,请等待本轮回复结束,或先点「停止」再发送新的问题。",
detail="Current thread already has an active run. Please wait or stop it before sending another message.",
) from exc
except UnsupportedStrategyError as exc:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise HTTPException(status_code=501, detail=str(exc)) from exc
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
t_post_create = time.perf_counter()
# Upsert thread metadata so the thread appears in /threads/search,
@@ -759,7 +983,12 @@ async def start_run(
logger.warning("Failed to upsert thread_meta for %s (non-fatal)", sanitize_log_param(thread_id))
t_post_thread_upsert = time.perf_counter()
agent_factory = resolve_agent_factory(body.assistant_id)
try:
agent_factory = resolve_agent_factory(body.assistant_id)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
graph_input = normalize_input(body.input)
# Admin-configurable prompt prefix: when the caller opted in via
@@ -797,24 +1026,38 @@ async def start_run(
stream_modes = normalize_stream_modes(body.stream_mode)
t_pre_task = time.perf_counter()
task = asyncio.create_task(
run_agent(
bridge,
run_mgr,
record,
ctx=run_ctx,
agent_factory=agent_factory,
graph_input=graph_input,
config=config,
stream_modes=stream_modes,
stream_subgraphs=body.stream_subgraphs,
interrupt_before=body.interrupt_before,
interrupt_after=body.interrupt_after,
# 透传 SDK ``Command``:resume / update / goto。Pregel 会据此从最近
# checkpoint 处恢复,而不是把 graph_input 当新一轮输入跑。
command=getattr(body, "command", None),
heartbeat_interval_seconds = 20
if concurrency_gate is not None:
heartbeat_interval_seconds = int(
getattr(getattr(concurrency_gate, "config", None), "heartbeat_interval_seconds", 20) or 20
)
)
try:
task = asyncio.create_task(
_run_agent_with_concurrency_lease(
concurrency_gate=concurrency_gate,
lease_acquired=redis_lease_acquired,
lease_metadata=lease_metadata,
heartbeat_interval_seconds=heartbeat_interval_seconds,
run_agent_kwargs={
"bridge": bridge,
"run_manager": run_mgr,
"record": record,
"ctx": run_ctx,
"agent_factory": agent_factory,
"graph_input": graph_input,
"config": config,
"stream_modes": stream_modes,
"stream_subgraphs": body.stream_subgraphs,
"interrupt_before": body.interrupt_before,
"interrupt_after": body.interrupt_after,
"command": getattr(body, "command", None),
},
)
)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
record.task = task
# Title sync is handled by worker.py's finally block which reads the