Merge branch 'master-dev' into 'master'

This commit is contained in:
吉鑫升 committed 2026-07-21 14:47:46 +08:00
commit 5436719bfb
39 files changed
+2395 -291

No files matched your search

@@ -11,8 +11,9 @@ from pydantic import BaseModel, ConfigDict, Field
from deerflow.config.acp_config import ACPAgentConfig, load_acp_config_from_dict
from deerflow.config.agents_api_config import AgentsApiConfig, load_agents_api_config_from_dict
from deerflow.config.browser_context_config import BrowserContextConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.checkpointer_config import CheckpointerConfig, load_checkpointer_config_from_dict
from deerflow.config.concurrency_config import ConcurrencyConfig
from deerflow.config.database_config import DatabaseConfig
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.gateway_config import GatewayConfig
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
@@ -137,8 +138,9 @@ class AppConfig(BaseModel):
gateway: GatewayConfig = Field(default_factory=GatewayConfig, description="Gateway FastAPI listening port (used by intra-process loopback)")
run_events: RunEventsConfig = Field(default_factory=RunEventsConfig, description="Run event storage configuration")
checkpointer: CheckpointerConfig | None = Field(default=None, description="Checkpointer configuration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
stream_bridge: StreamBridgeConfig | None = Field(default=None, description="Stream bridge configuration")
concurrency: ConcurrencyConfig = Field(default_factory=ConcurrencyConfig, description="Optional cross-worker concurrency coordination")
browser_context: BrowserContextConfig = Field(default_factory=BrowserContextConfig, description="Optional browser page capture integration")
knowledge: KnowledgeConfig = Field(default_factory=KnowledgeConfig, description="Product knowledge base (obsidian-wiki compatible vault) configuration")
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
@@ -0,0 +1,38 @@
"""Runtime concurrency coordination configuration."""
from __future__ import annotations
from pydantic import BaseModel, Field
class RedisConcurrencyConfig(BaseModel):
"""Optional Redis-backed cross-worker concurrency coordination."""
enabled: bool = Field(default=False, description="Enable Redis-backed global run concurrency coordination.")
url: str | None = Field(default=None, description="Redis URL, for example redis://127.0.0.1:6379/0.")
password: str | None = Field(
default=None,
description="Optional Redis password. You may also include credentials directly in url.",
)
namespace: str = Field(
default="cmzs_deerflow:concurrency",
description="Redis key prefix for concurrency leases. Use a deployment/system prefix when Redis is shared.",
)
socket_connect_timeout_ms: int = Field(default=80, ge=1, le=10_000)
socket_timeout_ms: int = Field(default=80, ge=1, le=10_000)
acquire_timeout_ms: int = Field(default=120, ge=1, le=10_000)
operation_timeout_ms: int = Field(default=80, ge=1, le=10_000)
circuit_breaker_failures: int = Field(default=3, ge=1, le=100)
circuit_breaker_cooldown_seconds: int = Field(default=30, ge=1, le=3600)
lease_ttl_seconds: int = Field(default=90, ge=10, le=3600)
heartbeat_interval_seconds: int = Field(default=20, ge=5, le=600)
fail_open: bool = Field(
default=True,
description="Allow runs to proceed with local-only limits when Redis is unavailable or slow.",
)
class ConcurrencyConfig(BaseModel):
"""Top-level concurrency coordination config."""
redis: RedisConcurrencyConfig = Field(default_factory=RedisConcurrencyConfig)
@@ -345,6 +345,16 @@ class ModelConcurrencySettings(BaseModel):
default_factory=dict,
description="Optional per-model overrides: model_name -> max concurrent runs.",
)
user_default_limit: int = Field(
default=0,
ge=0,
le=10_000,
description="Fallback max concurrent runs per user and model. 0 disables the per-user fallback.",
)
user_model_limits: dict[str, int] = Field(
default_factory=dict,
description="Optional per-user per-model overrides: model_name -> max concurrent runs for one user.",
)
model_config = ConfigDict(extra="ignore")
@@ -7,14 +7,19 @@ directly from ``deerflow.runtime``.
from .checkpointer import checkpointer_context, get_checkpointer, make_checkpointer, reset_checkpointer
from .runs import (
AcquireDecision,
AcquireResult,
ActiveLease,
ConflictError,
DisconnectMode,
ModelConcurrencyLimitError,
RedisConcurrencyGate,
RunContext,
RunManager,
RunRecord,
RunStatus,
UnsupportedStrategyError,
make_redis_concurrency_gate,
run_agent,
)
from .serialization import serialize, serialize_channel_values, serialize_lc_object, serialize_messages_tuple
@@ -28,14 +33,19 @@ __all__ = [
"make_checkpointer",
"reset_checkpointer",
# runs
"AcquireDecision",
"AcquireResult",
"ActiveLease",
"ConflictError",
"DisconnectMode",
"ModelConcurrencyLimitError",
"RedisConcurrencyGate",
"RunContext",
"RunManager",
"RunRecord",
"RunStatus",
"UnsupportedStrategyError",
"make_redis_concurrency_gate",
"run_agent",
# serialization
"serialize",
@@ -2,16 +2,22 @@
from .manager import ConflictError, ModelConcurrencyLimitError, RunManager, RunRecord, UnsupportedStrategyError
from .schemas import DisconnectMode, RunStatus
from .concurrency_gate import AcquireDecision, AcquireResult, ActiveLease, RedisConcurrencyGate, make_redis_concurrency_gate
from .worker import RunContext, run_agent
__all__ = [
"AcquireDecision",
"AcquireResult",
"ActiveLease",
"ConflictError",
"DisconnectMode",
"ModelConcurrencyLimitError",
"RedisConcurrencyGate",
"RunContext",
"RunManager",
"RunRecord",
"RunStatus",
"UnsupportedStrategyError",
"make_redis_concurrency_gate",
"run_agent",
]
@@ -0,0 +1,379 @@
"""Optional cross-worker run concurrency gates.
The Redis gate is deliberately fail-open by default: Redis coordinates global
limits when it is fast and healthy, but it must not block the chat path when it
is slow or unavailable.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass
from enum import Enum
from typing import Any
from deerflow.config.concurrency_config import RedisConcurrencyConfig
logger = logging.getLogger(__name__)
class AcquireDecision(str, Enum):
ACQUIRED = "acquired"
REJECTED = "rejected"
DEGRADED = "degraded"
@dataclass(frozen=True)
class AcquireResult:
decision: AcquireDecision
reason: str = ""
source: str = "redis"
@property
def acquired(self) -> bool:
return self.decision == AcquireDecision.ACQUIRED
@property
def rejected(self) -> bool:
return self.decision == AcquireDecision.REJECTED
@property
def degraded(self) -> bool:
return self.decision == AcquireDecision.DEGRADED
@dataclass(frozen=True)
class ActiveLease:
run_id: str
thread_id: str = ""
model_name: str = ""
user_id: str = ""
assistant_id: str = ""
created_at: float = 0.0
expires_at: float = 0.0
class CircuitBreaker:
"""Tiny in-process circuit breaker for Redis failures."""
def __init__(self, *, failure_threshold: int, cooldown_seconds: int) -> None:
self._failure_threshold = max(1, int(failure_threshold))
self._cooldown_seconds = max(1, int(cooldown_seconds))
self._failures = 0
self._opened_until = 0.0
def allow(self) -> bool:
return time.monotonic() >= self._opened_until
def record_success(self) -> None:
self._failures = 0
self._opened_until = 0.0
def record_failure(self) -> None:
self._failures += 1
if self._failures >= self._failure_threshold:
self._opened_until = time.monotonic() + self._cooldown_seconds
@property
def open(self) -> bool:
return not self.allow()
class RedisConcurrencyGate:
"""Redis lease-based global concurrency limiter."""
_ACQUIRE_SCRIPT = """
local now = tonumber(ARGV[1])
local expires_at = tonumber(ARGV[2])
local run_id = ARGV[3]
local model_limit = tonumber(ARGV[4])
local user_model_limit = tonumber(ARGV[5])
local reject_thread = tonumber(ARGV[6])
local payload = ARGV[7]
local ttl_ms = tonumber(ARGV[8])
for i = 1, 4 do
redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now)
end
if reject_thread == 1 and redis.call('ZCARD', KEYS[4]) > 0 then
return {'rejected', 'thread'}
end
if model_limit > 0 and redis.call('ZCARD', KEYS[2]) >= model_limit then
return {'rejected', 'model'}
end
if user_model_limit > 0 and redis.call('ZCARD', KEYS[3]) >= user_model_limit then
return {'rejected', 'user_model'}
end
redis.call('ZADD', KEYS[1], expires_at, run_id)
redis.call('ZADD', KEYS[2], expires_at, run_id)
if user_model_limit > 0 then
redis.call('ZADD', KEYS[3], expires_at, run_id)
end
redis.call('ZADD', KEYS[4], expires_at, run_id)
redis.call('SET', KEYS[5], payload, 'PX', ttl_ms)
return {'acquired', ''}
"""
def __init__(self, config: RedisConcurrencyConfig) -> None:
self.config = config
self._client: Any | None = None
self._breaker = CircuitBreaker(
failure_threshold=config.circuit_breaker_failures,
cooldown_seconds=config.circuit_breaker_cooldown_seconds,
)
@property
def enabled(self) -> bool:
return bool(self.config.enabled and self.config.url)
def health(self) -> dict[str, Any]:
return {
"enabled": self.enabled,
"source": "redis",
"circuit_open": self._breaker.open,
"fail_open": self.config.fail_open,
}
async def aclose(self) -> None:
client = self._client
self._client = None
if client is not None:
try:
await client.aclose()
except Exception:
logger.debug("Failed to close Redis concurrency client", exc_info=True)
def _ns(self) -> str:
return self.config.namespace.strip().rstrip(":") or "cmzs_deerflow:concurrency"
def _keys(self, *, run_id: str, model_name: str, user_id: str, thread_id: str) -> list[str]:
ns = self._ns()
return [
f"{ns}:all",
f"{ns}:model:{model_name}",
f"{ns}:user_model:{model_name}:{user_id}",
f"{ns}:thread:{thread_id}",
f"{ns}:run:{run_id}",
]
async def _redis(self) -> Any:
if self._client is not None:
return self._client
try:
from redis.asyncio import Redis
except Exception as exc: # pragma: no cover - exercised when dependency is absent
raise RuntimeError("redis package is not installed") from exc
timeout = max(0.001, self.config.socket_timeout_ms / 1000)
connect_timeout = max(0.001, self.config.socket_connect_timeout_ms / 1000)
self._client = Redis.from_url(
self.config.url,
password=self.config.password,
socket_timeout=timeout,
socket_connect_timeout=connect_timeout,
decode_responses=True,
)
return self._client
async def _with_timeout(self, coro, timeout_ms: int) -> Any:
return await asyncio.wait_for(coro, timeout=max(0.001, timeout_ms / 1000))
def _degraded(self, reason: str) -> AcquireResult:
return AcquireResult(AcquireDecision.DEGRADED, reason=reason)
def _on_failure(self, action: str, exc: BaseException) -> None:
self._breaker.record_failure()
logger.warning(
"Redis concurrency %s failed; fail_open=%s circuit_open=%s: %s",
action,
self.config.fail_open,
self._breaker.open,
exc,
)
async def try_acquire(
self,
*,
run_id: str,
thread_id: str,
model_name: str,
user_id: str,
assistant_id: str = "",
model_limit: int | None = None,
user_model_limit: int | None = None,
reject_thread: bool = True,
) -> AcquireResult:
if not self.enabled:
return self._degraded("disabled")
if not self._breaker.allow():
return self._degraded("circuit_open")
now = time.time()
expires_at = now + self.config.lease_ttl_seconds
payload = {
"run_id": run_id,
"thread_id": thread_id,
"model_name": model_name,
"user_id": user_id,
"assistant_id": assistant_id,
"created_at": now,
"expires_at": expires_at,
}
import json
keys = self._keys(run_id=run_id, model_name=model_name, user_id=user_id, thread_id=thread_id)
ttl_ms = int(self.config.lease_ttl_seconds * 1000)
try:
client = await self._redis()
raw = await self._with_timeout(
client.eval(
self._ACQUIRE_SCRIPT,
len(keys),
*keys,
str(now),
str(expires_at),
run_id,
str(max(0, int(model_limit or 0))),
str(max(0, int(user_model_limit or 0))),
"1" if reject_thread else "0",
json.dumps(payload, ensure_ascii=False),
str(ttl_ms),
),
self.config.acquire_timeout_ms,
)
self._breaker.record_success()
except Exception as exc:
self._on_failure("acquire", exc)
if self.config.fail_open:
return self._degraded(type(exc).__name__)
raise
status = raw[0] if isinstance(raw, list) and raw else ""
reason = raw[1] if isinstance(raw, list) and len(raw) > 1 else ""
if status == "acquired":
return AcquireResult(AcquireDecision.ACQUIRED)
return AcquireResult(AcquireDecision.REJECTED, reason=str(reason or "limit"))
async def heartbeat(
self,
*,
run_id: str,
thread_id: str,
model_name: str,
user_id: str,
user_model_accounted: bool = True,
) -> None:
if not self.enabled or not self._breaker.allow():
return
now = time.time()
expires_at = now + self.config.lease_ttl_seconds
keys = self._keys(run_id=run_id, model_name=model_name, user_id=user_id, thread_id=thread_id)
try:
client = await self._redis()
pipe = client.pipeline(transaction=False)
refresh_keys = [keys[0], keys[1], keys[3]]
if user_model_accounted:
refresh_keys.append(keys[2])
for key in refresh_keys:
pipe.zadd(key, {run_id: expires_at})
pipe.expire(keys[4], self.config.lease_ttl_seconds)
await self._with_timeout(pipe.execute(), self.config.operation_timeout_ms)
self._breaker.record_success()
except Exception as exc:
self._on_failure("heartbeat", exc)
async def release(
self,
*,
run_id: str,
thread_id: str,
model_name: str,
user_id: str,
user_model_accounted: bool = True,
) -> None:
if not self.enabled or not self._breaker.allow():
return
keys = self._keys(run_id=run_id, model_name=model_name, user_id=user_id, thread_id=thread_id)
try:
client = await self._redis()
pipe = client.pipeline(transaction=False)
release_keys = [keys[0], keys[1], keys[3]]
if user_model_accounted:
release_keys.append(keys[2])
for key in release_keys:
pipe.zrem(key, run_id)
pipe.delete(keys[4])
await self._with_timeout(pipe.execute(), self.config.operation_timeout_ms)
self._breaker.record_success()
except Exception as exc:
self._on_failure("release", exc)
async def count_active(self) -> int | None:
if not self.enabled or not self._breaker.allow():
return None
key = f"{self._ns()}:all"
now = time.time()
try:
client = await self._redis()
pipe = client.pipeline(transaction=False)
pipe.zremrangebyscore(key, "-inf", now)
pipe.zcard(key)
result = await self._with_timeout(pipe.execute(), self.config.operation_timeout_ms)
self._breaker.record_success()
return int(result[-1] or 0)
except Exception as exc:
self._on_failure("count", exc)
return None
async def list_active(self, *, limit: int = 1000) -> list[ActiveLease] | None:
if not self.enabled or not self._breaker.allow():
return None
import json
all_key = f"{self._ns()}:all"
now = time.time()
try:
client = await self._redis()
await self._with_timeout(client.zremrangebyscore(all_key, "-inf", now), self.config.operation_timeout_ms)
run_ids = await self._with_timeout(client.zrange(all_key, 0, max(0, limit - 1)), self.config.operation_timeout_ms)
if not run_ids:
self._breaker.record_success()
return []
keys = [f"{self._ns()}:run:{run_id}" for run_id in run_ids]
payloads = await self._with_timeout(client.mget(keys), self.config.operation_timeout_ms)
leases: list[ActiveLease] = []
for run_id, payload in zip(run_ids, payloads, strict=False):
if not payload:
continue
try:
data = json.loads(payload)
except Exception:
continue
leases.append(
ActiveLease(
run_id=str(data.get("run_id") or run_id),
thread_id=str(data.get("thread_id") or ""),
model_name=str(data.get("model_name") or ""),
user_id=str(data.get("user_id") or ""),
assistant_id=str(data.get("assistant_id") or ""),
created_at=float(data.get("created_at") or 0),
expires_at=float(data.get("expires_at") or 0),
)
)
self._breaker.record_success()
return leases
except Exception as exc:
self._on_failure("list", exc)
return None
def make_redis_concurrency_gate(config: Any) -> RedisConcurrencyGate | None:
concurrency = getattr(config, "concurrency", None)
redis_config = getattr(concurrency, "redis", None)
if redis_config is None or not getattr(redis_config, "enabled", False) or not getattr(redis_config, "url", None):
return None
return RedisConcurrencyGate(redis_config)
@@ -218,6 +218,8 @@ class RunManager:
kwargs: dict | None = None,
multitask_strategy: str = "reject",
model_concurrency_limit: tuple[str, int] | None = None,
user_model_concurrency_limit: tuple[str, str, int] | None = None,
run_id: str | None = None,
) -> RunRecord:
"""Atomically check for inflight runs and create a new one.
@@ -228,7 +230,7 @@ class RunManager:
This method holds the lock across both the check and the insert,
eliminating the TOCTOU race in separate ``has_inflight`` + ``create``.
"""
run_id = str(uuid.uuid4())
run_id = run_id or str(uuid.uuid4())
now = _now_iso()
_supported_strategies = ("reject", "interrupt", "rollback")
@@ -261,7 +263,20 @@ class RunManager:
model_name, limit = model_concurrency_limit
active_for_model = sum(1 for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running) and str((r.metadata or {}).get("model_name") or "").strip() == model_name)
if active_for_model >= max(1, int(limit)):
raise ModelConcurrencyLimitError(model_name, limit)
raise ModelConcurrencyLimitError(model_name, limit, scope="model")
if user_model_concurrency_limit is not None:
model_name, user_id, limit = user_model_concurrency_limit
active_for_user_model = sum(
1
for r in self._runs.values()
if r.status in (RunStatus.pending, RunStatus.running)
and str((r.metadata or {}).get("model_name") or "").strip() == model_name
and str((r.metadata or {}).get("user_id") or "").strip() == user_id
and str((r.metadata or {}).get("run_scope") or "external_root").strip() == "external_root"
)
if active_for_user_model >= max(1, int(limit)):
raise ModelConcurrencyLimitError(model_name, limit, scope="user_model", user_id=user_id)
record = RunRecord(
run_id=run_id,
@@ -316,9 +331,11 @@ class ConflictError(Exception):
class ModelConcurrencyLimitError(Exception):
"""Raised when a model-specific in-flight run limit has been reached."""
def __init__(self, model_name: str, limit: int) -> None:
def __init__(self, model_name: str, limit: int, *, scope: str = "model", user_id: str | None = None) -> None:
self.model_name = model_name
self.limit = limit
self.scope = scope
self.user_id = user_id
super().__init__(f"Model {model_name} has reached its concurrency limit")
@@ -0,0 +1,39 @@
"""Small in-memory gate used by legacy model-concurrency tests."""
from __future__ import annotations
import asyncio
class ModelConcurrencyGate:
"""Track per-model active counts with atomic try-acquire semantics."""
def __init__(self) -> None:
self._lock = asyncio.Lock()
self._counts: dict[str, int] = {}
async def try_acquire(self, model_name: str, limit: int) -> bool:
key = str(model_name).strip()
if not key:
return True
async with self._lock:
current = self._counts.get(key, 0)
if current >= max(1, int(limit)):
return False
self._counts[key] = current + 1
return True
async def release(self, model_name: str) -> None:
key = str(model_name).strip()
if not key:
return
async with self._lock:
current = self._counts.get(key, 0)
if current <= 1:
self._counts.pop(key, None)
else:
self._counts[key] = current - 1
async def snapshot(self) -> dict[str, int]:
async with self._lock:
return dict(self._counts)
@@ -23,9 +23,10 @@ dependencies = [
"markdownify>=1.2.2",
"markitdown[all,xlsx]>=0.0.1a2",
"pydantic>=2.12.5",
"pyyaml>=6.0.3",
"readabilipy>=0.3.0",
"tavily-python>=0.7.17",
"pyyaml>=6.0.3",
"readabilipy>=0.3.0",
"redis>=5.0.0",
"tavily-python>=0.7.17",
"firecrawl-py>=1.15.0",
"tiktoken>=0.8.0",
"ddgs>=9.10.0",