sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -0,0 +1 @@
"""Single-host managed online workers; background scheduling stays on the gateway."""
@@ -0,0 +1,206 @@
"""Bound executing runs, including detached/background runs, not HTTP streams."""
from __future__ import annotations
import asyncio
from contextvars import ContextVar
from functools import wraps
from fastapi import HTTPException
_worker_ticket = ContextVar("worker_admission_ticket", default=None)
class AdmissionTicket:
def __init__(self, gate):
if gate.active + gate.waiting >= gate.capacity + gate.queue_limit:
raise HTTPException(429, "执行节点繁忙,请稍后重试", headers={"Retry-After": "3"})
self.gate = gate
self.deadline = asyncio.get_running_loop().time() + gate.timeout
self.waiting = True
self.acquired = False
self.released = False
self.started = False
gate.waiting += 1
async def acquire(self):
try:
remaining = max(0.001, self.deadline - asyncio.get_running_loop().time())
await asyncio.wait_for(self.gate.semaphore.acquire(), remaining)
self.acquired = True
self.gate.active += 1
except TimeoutError:
raise HTTPException(429, "等待执行超时,请稍后重试") from None
finally:
if self.waiting:
self.gate.waiting -= 1
self.waiting = False
def release(self, _task=None):
if self.released:
return
self.released = True
if self.waiting:
self.gate.waiting -= 1
self.waiting = False
if self.acquired:
self.gate.active -= 1
self.gate.semaphore.release()
async def _finish_queued_run(kwargs, *, cancelled=False, message="等待执行超时,请稍后重试"):
from deerflow.runtime.runs.schemas import RunStatus
manager, record, bridge = kwargs["run_manager"], kwargs["record"], kwargs["bridge"]
try:
async with asyncio.timeout(5):
await manager.set_status(record.run_id, RunStatus.interrupted if cancelled else RunStatus.error, error=None if cancelled else message)
ctx = kwargs.get("ctx")
store = getattr(ctx, "thread_store", None)
if store is not None and not getattr(record, "superseded", False):
await store.update_status(record.thread_id, "interrupted" if cancelled else "error")
finally:
try:
if not cancelled:
await bridge.publish(record.run_id, "error", {"name": "WorkerQueueTimeout", "message": message})
finally:
await bridge.publish_end(record.run_id)
asyncio.create_task(bridge.cleanup(record.run_id, delay=60))
async def run_with_worker_admission(runner, **kwargs):
ticket = _worker_ticket.get()
if ticket is None:
return await runner(**kwargs)
ticket.started = True
record, bridge = kwargs["record"], kwargs["bridge"]
# Publish a real, persisted run id before waiting for an execution slot.
# The router keeps its canonical Location/reconnect headers unchanged.
try:
await bridge.publish(record.run_id, "metadata", {"run_id": record.run_id, "thread_id": record.thread_id,
"worker_queue": {"waiting": ticket.gate.waiting, "capacity": ticket.gate.capacity}})
await ticket.acquire()
except asyncio.CancelledError:
await _finish_queued_run(kwargs, cancelled=True)
raise
except HTTPException as exc:
await _finish_queued_run(kwargs, message=str(exc.detail))
return
except Exception:
await _finish_queued_run(kwargs, message="执行节点排队初始化失败,请查看节点日志")
return
return await runner(**kwargs)
class Admission:
def __init__(self, capacity: int, queue_limit: int = 128, timeout: float = 60):
self.capacity = capacity
self.active = 0
self.waiting = 0
self.queue_limit = queue_limit
self.timeout = timeout
self.semaphore = asyncio.Semaphore(capacity)
self.lock = asyncio.Lock()
async def acquire(self):
async with self.lock:
if self.waiting >= self.queue_limit:
raise HTTPException(429, "执行节点繁忙,请稍后重试", headers={"Retry-After": "3"})
self.waiting += 1
try:
await asyncio.wait_for(self.semaphore.acquire(), self.timeout)
except TimeoutError:
raise HTTPException(429, "等待执行超时,请稍后重试", headers={"Retry-After": "3"}) from None
finally:
self.waiting -= 1
self.active += 1
def release(self, _task=None):
self.active -= 1
self.semaphore.release()
def bounded_worker_run(function):
@wraps(function)
async def wrapped(body, thread_id, request, *args, **kwargs):
pool = getattr(request.app.state, "worker_pool", None)
fallback_to_main = bool(getattr(getattr(request, "state", None), "worker_pool_fallback", False))
if pool is not None and pool.db is not None and pool.db.execute("SELECT 1 FROM affinity WHERE thread=?", (thread_id,)).fetchone() and not fallback_to_main:
raise HTTPException(409, "该会话由独立 Worker 管理,请通过会话接口提交,不能在主服务重复执行")
gate = getattr(request.app.state, "worker_admission", None)
if gate is None:
if pool is None:
return await function(body, thread_id, request, *args, **kwargs)
if pool.local_threads.get(thread_id, 0):
raise HTTPException(409, "该会话正在主服务执行,请等待当前回答完成")
# Reserve before the first await, including start_run's preparation
# phase before RunRecord.task exists. Otherwise a concurrent routed
# request could assign this same thread to another process.
pool.local_threads[thread_id] = pool.local_threads.get(thread_id, 0) + 1
def release_local(_task=None):
remaining = pool.local_threads.get(thread_id, 1) - 1
if remaining:
pool.local_threads[thread_id] = remaining
else:
pool.local_threads.pop(thread_id, None)
try:
record = await function(body, thread_id, request, *args, **kwargs)
except BaseException:
release_local()
raise
if record.task is None:
release_local()
else:
record.task.add_done_callback(release_local)
return record
if getattr(request.app.state, "worker_deferred_admission", False) and getattr(request.app.state, "concurrency_gate", None) is None:
ticket = AdmissionTicket(gate)
token = _worker_ticket.set(ticket)
try:
from deerflow.runtime.thread_paths import aresolve_path_user_id
await aresolve_path_user_id(thread_id)
refresh = getattr(request.app.state, "worker_skill_refresh", None)
if refresh is not None:
await refresh.check()
record = await function(body, thread_id, request, *args, **kwargs)
except BaseException:
ticket.release()
raise
finally:
_worker_ticket.reset(token)
if record.task is None:
ticket.release()
else:
def finish_ticket(task):
ticket.release()
if task.cancelled() and not ticket.started:
manager = getattr(request.app.state, "run_manager", None)
bridge = getattr(request.app.state, "stream_bridge", None)
if manager is not None and bridge is not None:
asyncio.create_task(_finish_queued_run({"run_manager": manager, "record": record, "bridge": bridge}, cancelled=True))
record.task.add_done_callback(finish_ticket)
return record
await gate.acquire()
try:
# File/path middleware is synchronous. Resolve the persisted owner
# on the database's home loop before it touches those helpers.
from deerflow.runtime.thread_paths import aresolve_path_user_id
await aresolve_path_user_id(thread_id)
refresh = getattr(request.app.state, "worker_skill_refresh", None)
if refresh is not None:
await refresh.check()
record = await function(body, thread_id, request, *args, **kwargs)
except BaseException:
gate.release()
raise
if record.task is None:
gate.release()
else:
record.task.add_done_callback(gate.release)
return record
return wrapped
@@ -0,0 +1,359 @@
"""One gateway owns reconciliation and durable conversation affinity.
Never move conversations from a live worker. A draining worker is stopped only
after its run registry AND outstanding HTTP requests are empty. Failure does
not replay a request, because a model/tool may already have produced effects.
"""
from __future__ import annotations
import asyncio
import json
import os
import sqlite3
import time
from urllib.parse import quote
import httpx
from .docker import DockerEngine, DockerError, worker_spec
from .settings import SettingsStore
class PoolUnavailable(RuntimeError):
pass
class Controller:
def __init__(self, store: SettingsStore):
self.store = store
self.lock = asyncio.Lock()
self.route_lock = asyncio.Lock()
self.nodes: dict[str, dict] = {}
self.error: str | None = None
self.task: asyncio.Task | None = None
self._lock_handle = None
self.http = httpx.AsyncClient(timeout=httpx.Timeout(15, read=None), trust_env=False, limits=httpx.Limits(max_connections=256, max_keepalive_connections=64))
self.db: sqlite3.Connection | None = None
self.pending = 0
self.inflight: dict[str, int] = {}
self.local_threads: dict[str, int] = {}
self._python_probe_key: tuple | None = None
self._python_probe_result: str | None = None
self._worker_image_key: tuple | None = None
self._worker_image_result: str | None = None
self.discovery_complete = not bool(store.load().source_container)
def _ensure_db(self):
if self.db is None:
self.db = sqlite3.connect(self.store.directory / "affinity.sqlite3")
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("CREATE TABLE IF NOT EXISTS affinity (thread TEXT PRIMARY KEY, node TEXT NOT NULL)")
self.db.commit()
return self.db
def _acquire_controller_lock(self):
if self._lock_handle is not None:
return
self.store.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
handle = (self.store.directory / "controller.lock").open("a+b")
try:
if os.name == "nt":
import msvcrt
handle.seek(0)
if not handle.read(1):
handle.write(b"0")
handle.flush()
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise RuntimeError("Worker 池只能由一个主服务进程管理;请关闭重复的 Worker 池控制器") from None
self._lock_handle = handle
def start(self):
settings = self.store.load()
if settings.enabled:
self._acquire_controller_lock()
self._ensure_db()
if self.task is None or self.task.done():
self.task = asyncio.create_task(self.loop())
if hasattr(self, "diagnostics") and self.diagnostics.enabled:
self.diagnostics.start()
async def close(self):
if self.task:
self.task.cancel()
await asyncio.gather(self.task, return_exceptions=True)
if hasattr(self, "diagnostics"):
await self.diagnostics.close()
await self.http.aclose()
if self.db is not None:
self.db.close()
self.db = None
self._release_controller_lock()
async def loop(self):
try:
while True:
try:
await self.reconcile()
except Exception as exc:
self.error = str(exc) if isinstance(exc, (DockerError, PoolUnavailable)) else f"节点检查失败({type(exc).__name__})"
settings = self.store.load()
if not settings.enabled and not self.nodes:
return
await asyncio.sleep(5)
finally:
if not self.store.load().enabled:
self._release_controller_lock()
def _release_controller_lock(self):
if self._lock_handle is None:
return
if os.name == "nt":
import msvcrt
self._lock_handle.seek(0)
msvcrt.locking(self._lock_handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(self._lock_handle, fcntl.LOCK_UN)
self._lock_handle.close()
self._lock_handle = None
async def inspect_template(self, settings):
config = settings
if config.desired * config.cpu > config.total_cpu or config.desired * config.memory_mb > config.total_memory_mb:
raise PoolUnavailable("Worker 总资源超过设置的预算,请减少数量或调整预算")
if config.desired * (config.database_pool_size + 2 * config.postgres_pool_size) > config.database_connection_budget:
raise PoolUnavailable("Worker 数据库连接总额度超过预算,请减少数量或单节点连接数")
if not config.source_container or not config.network:
raise PoolUnavailable("请填写源容器名称和 Docker 网络")
engine = DockerEngine(config)
source = await asyncio.to_thread(engine.request, "GET", f"/containers/{quote(config.source_container, safe='')}/json?size=true")
if source.get("Config", {}).get("Labels", {}).get("deerflow.worker-pool"):
raise PoolUnavailable("源容器必须是主服务,不能选择已有 Worker")
backend = await asyncio.to_thread(
engine.configured_storage_backend, source, config.python_executable, config.working_dir,
)
probe_key = (
config.transport, config.host, config.port, config.username, config.source_container,
source.get("Id"), source.get("Image"), config.python_executable,
config.working_dir, source.get("Config", {}).get("User", ""), backend,
)
if probe_key == self._python_probe_key and self._python_probe_result:
python_executable = self._python_probe_result
else:
python_executable = await asyncio.to_thread(
engine.discover_worker_python, source, config.python_executable, config.working_dir,
)
self._python_probe_key = probe_key
self._python_probe_result = python_executable
if python_executable != config.python_executable:
config = config.model_copy(update={"python_executable": python_executable})
self.store.save(config)
self._python_probe_key = (
config.transport, config.host, config.port, config.username, config.source_container,
source.get("Id"), source.get("Image"), config.python_executable,
config.working_dir, source.get("Config", {}).get("User", ""), backend,
)
runtime_image = None
if config.enabled and config.desired > 0:
image_key = (source.get("Id"), source.get("Image"), python_executable, backend)
if image_key != self._worker_image_key or not self._worker_image_result:
self._worker_image_result = await asyncio.to_thread(
engine.snapshot_worker_runtime, source, self.store.raw()["instance"],
)
self._worker_image_key = image_key
runtime_image = self._worker_image_result
spec = worker_spec(
source, config, self.store.raw()["instance"], self.store.raw()["token"],
str(self.store.directory), runtime_image=runtime_image,
require_shared_archive=backend == "postgres" and config.enabled and config.desired > 0,
)
return engine, spec
async def reconcile(self):
async with self.lock:
settings = self.store.load()
if not settings.enabled and not settings.source_container:
return
db = self._ensure_db()
engine, spec = await self.inspect_template(settings)
instance = self.store.raw()["instance"]
filters = quote(json.dumps({"label": [f"deerflow.worker-pool={instance}"]}))
containers = await asyncio.to_thread(engine.request, "GET", f"/containers/json?all=1&filters={filters}")
self.discovery_complete = True
desired = settings.desired if settings.enabled else 0
wanted = {f"dfw-{instance}-{i}" for i in range(desired)}
existing = {row["Names"][0].lstrip("/"): row for row in containers}
stale_templates = set()
for name, row in list(existing.items()):
self.nodes.setdefault(name, {"name": name, "id": row["Id"], "ready": False, "active": 0, "draining": name not in wanted, "url": f"http://{name}:8001"})
if name in wanted and row.get("State") == "running" and row.get("ImageID") and row["ImageID"] != spec["Image"]:
stale_templates.add(name)
self.nodes[name]["draining"] = True
# A stopped container cannot be executing a request. If its saved
# startup template has an old Python path, replace it with the
# resolved template instead of restarting the same broken command.
for name, row in list(existing.items()):
if name not in wanted or row["State"] == "running":
continue
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{quote(row['Id'], safe='')}/json")
worker_config = detail.get("Config", {})
stale_template = (
detail.get("Image") != spec["Image"]
or worker_config.get("Entrypoint") != spec["Entrypoint"]
or worker_config.get("WorkingDir") != spec["WorkingDir"]
)
if not stale_template:
continue
if hasattr(self, "diagnostics"):
await self.diagnostics.archive_logs(engine, name)
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{quote(row['Id'], safe='')}")
db.execute("DELETE FROM affinity WHERE node=?", (name,))
db.commit()
existing.pop(name, None)
self.nodes.pop(name, None)
for name, row in list(existing.items()):
previous = self.nodes.get(name, {})
node = {
"name": name,
"id": row["Id"],
"ready": previous.get("ready", False),
"ever_ready": previous.get("ever_ready", False),
"active": previous.get("active", 0),
"capacity": previous.get("capacity"),
"draining": name not in wanted or name in stale_templates,
"url": f"http://{name}:8001",
"checked_at": time.time(),
}
for key in ("running", "waiting", "process_started_at", "runtime_revision", "event_loop_lag_ms", "http"):
if key in previous:
node[key] = previous[key]
self.nodes[name] = node
if row["State"] != "running":
node["ready"] = False
node["ever_ready"] = False
# A confirmed stopped process cannot keep executing. Old
# conversations can move only after this confirmation.
db.execute("DELETE FROM affinity WHERE node=?", (name,))
db.commit()
if name in wanted:
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/start")
else:
if hasattr(self, "diagnostics"):
await self.diagnostics.archive_logs(engine, name)
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{row['Id']}")
self.nodes.pop(name, None)
continue
try:
diagnostics_enabled = bool(getattr(getattr(self, "diagnostics", None), "enabled", False))
response = await self.http.get(
node["url"] + "/internal/worker-status",
headers={
"x-deerflow-worker-token": self.store.raw()["token"],
"x-deerflow-diagnostics-enabled": "true" if diagnostics_enabled else "false",
},
timeout=3,
)
response.raise_for_status()
status = response.json()
node.update(ready=status["ready"], active=status["active"], capacity=status["capacity"], ever_ready=True)
for key in ("running", "waiting", "process_started_at", "runtime_revision", "event_loop_lag_ms", "http"):
if key in status:
node[key] = status[key]
except (httpx.HTTPError, ValueError, KeyError):
node["ready"] = False
node["error"] = "等待就绪或连接失败;请检查主容器是否位于同一网络、镜像代码和聊天存储配置"
continue
if node["draining"] and not node["active"] and not self.inflight.get(name, 0):
# Refuse all new reservations while stop is in flight.
node["ready"] = False
node["ever_ready"] = False
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/stop?t=30")
if hasattr(self, "diagnostics"):
await self.diagnostics.archive_logs(engine, name)
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{row['Id']}")
db.execute("DELETE FROM affinity WHERE node=?", (name,))
db.commit()
self.nodes.pop(name, None)
existing.pop(name, None)
# Replacements are created only after the previous container has
# drained and been removed. This also refreshes Workers after the
# main service's runtime image or configured storage backend changes.
for name in sorted(wanted - existing.keys()):
row = await asyncio.to_thread(engine.request, "POST", f"/containers/create?name={name}", spec)
self.nodes[name] = {
"name": name,
"id": row["Id"],
"ready": False,
"active": 0,
"draining": False,
"url": f"http://{name}:8001",
}
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/start")
existing[name] = {"Id": row["Id"], "State": "running", "ImageID": spec["Image"]}
# Missing nodes are not silently reassigned: an operator must
# reconcile the actual container state before changing affinity.
self.error = None
async def reserve(self, thread: str, *, create: bool) -> dict | None:
settings = self.store.load()
if not settings.enabled and not self.nodes:
return None
db = self._ensure_db()
# Container creation/SSH can be slow; routing must never wait on that lock.
async with self.route_lock:
row = db.execute("SELECT node FROM affinity WHERE thread=?", (thread,)).fetchone()
if row is None and create and self.local_threads.get(thread, 0):
raise PoolUnavailable("该会话正在主服务执行,请等待当前轮完成")
if row:
node = self.nodes.get(row[0])
if node and node.get("draining") and create:
return {"_fallback_to_main": True, "_fallback_reason": "worker_draining", "_worker": dict(node)}
# A delayed health probe stops new work, but must not hide an
# existing run's result. Keep reads/cancellation on its owner;
# stopped containers clear ever_ready before affinity changes.
if not node or (not node["ready"] and (create or not node.get("ever_ready"))):
raise PoolUnavailable("该会话的执行节点暂不可用,请稍后重试;系统不会重复提交任务")
if node["draining"] and create:
raise PoolUnavailable("该会话节点正在停止接单,请等待当前任务完成后重试")
if create:
capacity = max(1, int(node.get("capacity") or settings.concurrency))
load = max(int(node.get("active") or 0), self.inflight.get(node["name"], 0))
if load >= capacity:
return {"_fallback_to_main": True, "_fallback_reason": "worker_saturated", "_worker": dict(node)}
elif not settings.enabled or not create:
return None
else:
choices = [n for n in self.nodes.values() if n["ready"] and not n["draining"]]
if not choices:
return {"_fallback_to_main": True, "_fallback_reason": "workers_unavailable"}
available = []
for candidate in choices:
capacity = max(1, int(candidate.get("capacity") or settings.concurrency))
load = max(int(candidate.get("active") or 0), self.inflight.get(candidate["name"], 0))
if load < capacity:
available.append((load, candidate))
if not available:
return {"_fallback_to_main": True, "_fallback_reason": "worker_pool_saturated"}
_, node = min(available, key=lambda item: (item[0], item[1]["name"]))
db.execute("INSERT INTO affinity VALUES (?, ?)", (thread, node["name"]))
db.commit()
self.inflight[node["name"]] = self.inflight.get(node["name"], 0) + 1
return dict(node)
def release(self, name):
self.inflight[name] = max(0, self.inflight.get(name, 0) - 1)
def status(self):
return {"settings": self.store.public(), "nodes": [{k: v for k, v in n.items() if k != "url"} for n in self.nodes.values()], "error": self.error}
@@ -0,0 +1,348 @@
"""Admin-enabled, bounded diagnostics with independent, short DB probes."""
import asyncio
import json
import os
import re
import sqlite3
import time
from collections import deque
from urllib.parse import quote, urlencode
from .docker import DockerEngine, DockerError, decode_docker_output
from .host_metrics import rates
from .telemetry import RequestRecorder
def redact_logs(value):
value = re.sub(r"(?i)([a-z][a-z0-9+.-]*://)[^\s/@]+:[^\s/@]+@", r"\1[REDACTED]@", value)
value = re.sub(r"(?i)(Bearer\s+)[^\s\"']+", r"\1[REDACTED]", value)
value = re.sub(r"(?i)((?:password|passwd|api[_-]?key|access[_-]?token|secret|authorization|token)[\"']?\s*[:=]\s*[\"']?)[^\s,\"'}]+", r"\1[REDACTED]", value)
value = re.sub(r"\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+", "[REDACTED_JWT]", value)
value = re.sub(r"\bsk-[a-zA-Z0-9_-]{12,}", "[REDACTED_KEY]", value)
return value
def container_metrics(detail, stats, previous_stats=None):
cpu = stats.get("cpu_stats", {})
previous = previous_stats.get("cpu_stats", {}) if previous_stats else stats.get("precpu_stats", {})
delta = cpu.get("cpu_usage", {}).get("total_usage", 0) - previous.get("cpu_usage", {}).get("total_usage", 0)
system_delta = cpu.get("system_cpu_usage", 0) - previous.get("system_cpu_usage", 0)
cores = cpu.get("online_cpus") or len(cpu.get("cpu_usage", {}).get("percpu_usage", [])) or 1
memory = stats.get("memory_stats", {})
cache = memory.get("stats", {}).get("inactive_file", memory.get("stats", {}).get("total_inactive_file", 0))
state = detail.get("State", {})
return {
"name": detail.get("Name", "").lstrip("/"),
"image": detail.get("Config", {}).get("Image", ""),
"image_id": detail.get("Image", "")[:22],
"created_at": detail.get("Created"),
"started_at": state.get("StartedAt"),
"status": state.get("Status"),
"health": state.get("Health", {}).get("Status"),
"restarts": detail.get("RestartCount", 0),
"oom_killed": state.get("OOMKilled", False),
"cpu_percent": round(delta / system_delta * cores * 100, 2) if previous.get("system_cpu_usage") and system_delta > 0 and delta >= 0 else None,
"cpu_limit": detail.get("HostConfig", {}).get("NanoCpus", 0) / 1e9 or None,
"memory_bytes": max(0, memory.get("usage", 0) - cache),
"memory_limit": memory.get("limit"),
"pids": stats.get("pids_stats", {}).get("current"),
"log_policy": detail.get("HostConfig", {}).get("LogConfig", {}),
"host_proc_mounted": any(m.get("Destination") == "/host/proc" for m in detail.get("Mounts", [])),
"docker_socket_mounted": any(m.get("Destination") == "/var/run/docker.sock" for m in detail.get("Mounts", [])),
}
class Diagnostics:
def __init__(self, pool, app):
self.pool = pool
self.app = app
self.path = pool.store.directory / "diagnostics-settings.json"
self.archive_directory = pool.store.directory / "log-archive"
self.enabled = json.loads(self.path.read_text()).get("enabled", False) if self.path.exists() else False
self.recorder = RequestRecorder()
self.recorder.enabled = self.enabled
app.state.request_recorder = self.recorder
self.history = deque(maxlen=720)
self.latest = None
self.previous = None
self.container_previous = {}
from .revision import code_revision
self.runtime_revision = code_revision()
self.task = None
self.lag_task = None
self.loop_lag_ms = 0
self.db_engines = {}
self.db = None
if self.enabled:
self._ensure_db()
def _ensure_db(self):
if self.db is None:
self.db = sqlite3.connect(self.pool.store.directory / "diagnostics.sqlite3")
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("CREATE TABLE IF NOT EXISTS snapshots (id INTEGER PRIMARY KEY, at REAL, body TEXT)")
self.db.execute("CREATE TABLE IF NOT EXISTS requests (id INTEGER PRIMARY KEY, at REAL, body TEXT)")
for (value,) in reversed(self.db.execute("SELECT body FROM snapshots ORDER BY id DESC LIMIT 720").fetchall()):
self.history.append(json.loads(value))
for (value,) in reversed(self.db.execute("SELECT body FROM requests ORDER BY id DESC LIMIT 2000").fetchall()):
self.recorder.completed.append(json.loads(value))
return self.db
def start(self):
if not self.enabled:
return
self._ensure_db()
if self.task is not None and not self.task.done():
return
self.task = asyncio.create_task(self.loop())
self.lag_task = asyncio.create_task(self.measure_lag())
async def close(self):
for task in (self.task, self.lag_task):
if task:
task.cancel()
await asyncio.gather(*(t for t in (self.task, self.lag_task) if t), return_exceptions=True)
self.persist_requests()
if self.db is not None:
self.db.close()
self.db = None
for engine in self.db_engines.values():
await engine.dispose()
async def set_enabled(self, enabled):
self.enabled = enabled
self.recorder.enabled = enabled
temporary = self.path.with_suffix(".tmp")
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w") as stream:
json.dump({"enabled": enabled}, stream)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, self.path)
if enabled:
self._ensure_db()
self.start()
else:
tasks = [task for task in (self.task, self.lag_task) if task]
for task in tasks:
if task:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
self.task = None
self.lag_task = None
self.previous = None
self.persist_requests()
async def measure_lag(self):
while True:
before = asyncio.get_running_loop().time()
await asyncio.sleep(1)
self.loop_lag_ms = round(max(0, asyncio.get_running_loop().time() - before - 1) * 1000, 2) if self.enabled else 0
async def loop(self):
while True:
if self.enabled:
try:
await self.sample()
except Exception as exc:
self.latest = {"at": time.time(), "error": f"采集失败({type(exc).__name__})"}
await asyncio.sleep(5)
def persist_requests(self):
batch = []
while self.recorder.pending:
row = self.recorder.pending.popleft()
batch.append((row["started"], json.dumps(row)))
if batch and self.db is not None:
self.db.executemany("INSERT INTO requests(at, body) VALUES (?, ?)", batch)
self.db.execute("DELETE FROM requests WHERE id <= (SELECT COALESCE(MAX(id),0)-10000 FROM requests)")
self.db.commit()
async def sample(self):
self._ensure_db()
settings = self.pool.store.load()
engine = DockerEngine(settings)
names = [settings.source_container, *list(self.pool.nodes)]
async def read_container(name):
try:
encoded = quote(name, safe="")
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/json")
stats = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/stats?stream=false&one-shot=true")
key = detail.get("Id", name)
result = container_metrics(detail, stats, self.container_previous.get(key))
self.container_previous[key] = stats
return result
except Exception as exc:
return {"name": name, "error": f"节点指标不可用({type(exc).__name__})"}
async def host():
try:
raw = await asyncio.to_thread(engine.host_snapshot, settings.source_container)
result = rates(raw, self.previous)
self.previous = raw
return result
except Exception as exc:
detail = str(exc) if isinstance(exc, DockerError) else type(exc).__name__
return {"available": False, "message": f"宿主机采集不可用:{detail}。请检查只读挂载和 Docker 连接"}
host_data, containers, databases = await asyncio.gather(host(), asyncio.gather(*(read_container(n) for n in names if n)), self.database_metrics())
result = {"at": time.time(), "host": host_data, "containers": containers, "databases": databases, "event_loop_lag_ms": self.loop_lag_ms, "nodes": self.pool.status()["nodes"], "runtime_revision": self.runtime_revision}
self.latest = result
summary = {
"at": result["at"],
"cpu_percent": host_data.get("cpu_percent"),
"iowait_percent": host_data.get("iowait_percent"),
"memory_used_percent": round(100 * (1 - host_data["memory_available"] / host_data["memory_total"]), 2) if host_data.get("memory_total") else None,
"rx_bps": sum(v.get("rx_bps") or 0 for v in host_data.get("interfaces", {}).values()),
"tx_bps": sum(v.get("tx_bps") or 0 for v in host_data.get("interfaces", {}).values()),
"loop_lag_ms": self.loop_lag_ms,
"mysql_connections": databases.get("mysql", {}).get("connected"),
"postgres_connections": databases.get("postgres", {}).get("connected"),
"p95_headers_ms": self.recorder.summary()["p95_headers_ms"],
}
self.history.append(summary)
self.db.execute("INSERT INTO snapshots(at,body) VALUES (?,?)", (result["at"], json.dumps(summary)))
self.db.execute("DELETE FROM snapshots WHERE id <= (SELECT COALESCE(MAX(id),0)-720 FROM snapshots)")
self.db.commit()
self.persist_requests()
def status(self):
return {"enabled": self.enabled, "sample_interval_seconds": 5, "retained_samples": 720, "latest": self.latest, "history": list(self.history), "http": self.recorder.summary()}
async def database_metrics(self):
config = getattr(self.app.state, "config", None)
db = getattr(config, "database", None)
checkpoint = getattr(config, "checkpointer", None)
result = {}
if getattr(db, "backend", None) == "mysql":
result["mysql"] = await self.mysql_metrics(db.app_sqlalchemy_url)
else:
result["mysql"] = {"available": False, "reason": "业务库未使用 MySQL"}
from .preflight import postgres_connection_string
pg_url = postgres_connection_string(config)
if pg_url:
result["postgres"] = await self.postgres_metrics(pg_url)
else:
result["postgres"] = {"available": False, "configured": False, "reason": "当前聊天存储为 SQLite,未启用 PostgreSQL,不能显示为 0 个连接"}
result["chat_backend"] = "postgres" if pg_url else ("sqlite" if checkpoint and checkpoint.type in {"sqlite", "postgres"} else "unknown")
return result
async def mysql_metrics(self, url):
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
key = "mysql"
try:
if key not in self.db_engines:
self.db_engines[key] = create_async_engine(url, pool_size=1, max_overflow=0, pool_timeout=1, pool_pre_ping=False, connect_args={"connect_timeout": 2})
async with asyncio.timeout(4):
async with self.db_engines[key].connect() as connection:
statuses = dict((await connection.execute(text("SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Max_used_connections','Slow_queries','Aborted_connects')"))).all())
variables = dict((await connection.execute(text("SHOW GLOBAL VARIABLES WHERE Variable_name IN ('max_connections','long_query_time')"))).all())
rows = (await connection.execute(text("SHOW PROCESSLIST"))).mappings().all()
running = [r for r in rows if str(r.get("Command", "")).lower() not in {"sleep", "daemon"}]
return {
"available": True,
"connected": int(statuses.get("Threads_connected", 0)),
"running": int(statuses.get("Threads_running", 0)),
"max_connections": int(variables.get("max_connections", 0)),
"peak_connections": int(statuses.get("Max_used_connections", 0)),
"slow_queries": int(statuses.get("Slow_queries", 0)),
"slow_query_threshold_seconds": float(variables.get("long_query_time", 0)),
"longest_active_seconds": max((int(r.get("Time", 0)) for r in running), default=0),
"processes_visible": len(rows),
"note": "连接总数为 MySQL 全局值(包含采集连接);最长执行时间只覆盖当前账号可见进程,不读取 SQL 文本。",
}
except Exception as exc:
return {"available": False, "error": f"MySQL 指标不可用({type(exc).__name__}),请检查连接与只读监控权限"}
async def postgres_metrics(self, url):
try:
import psycopg
async with asyncio.timeout(4):
async with await psycopg.AsyncConnection.connect(url, connect_timeout=2, autocommit=True) as connection:
async with connection.cursor() as cursor:
query = (
"SELECT count(*), count(*) FILTER (WHERE state='active'), "
"count(*) FILTER (WHERE wait_event_type='Lock'), "
"COALESCE(max(EXTRACT(EPOCH FROM now()-query_start)) "
"FILTER (WHERE state='active' AND pid<>pg_backend_pid()), 0) "
"FROM pg_stat_activity"
)
await cursor.execute(query)
connected, active, locks, longest = await cursor.fetchone()
await cursor.execute("SHOW max_connections")
maximum = (await cursor.fetchone())[0]
return {
"available": True,
"configured": True,
"connected": connected,
"active": active,
"lock_waits": locks,
"max_connections": int(maximum),
"longest_active_seconds": float(longest),
"note": "按 pg_stat_activity 采集,普通账号可能看不到其他账号的状态和耗时;包含采集连接,不读取 SQL 文本。",
}
except Exception as exc:
return {"available": False, "configured": True, "error": f"PostgreSQL 指标不可用({type(exc).__name__}),请检查连接与监控权限"}
async def logs(self, name, tail=200, minutes=60, query=""):
if name.startswith("archive:"):
return self.read_archive(name[len("archive:") :], query)
settings = self.pool.store.load()
if name != settings.source_container and name not in self.pool.nodes:
raise DockerError("只能查看当前主服务和本 Worker 池的日志")
engine = DockerEngine(settings)
encoded = quote(name, safe="")
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/json")
if name != settings.source_container and detail.get("Config", {}).get("Labels", {}).get("deerflow.worker-pool") != self.pool.store.raw()["instance"]:
raise DockerError("节点不属于当前 Worker 池")
params = urlencode({"stdout": 1, "stderr": 1, "timestamps": 1, "tail": min(1000, max(1, tail)), "since": int(time.time() - min(1440, max(1, minutes)) * 60)})
data = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/logs?{params}", raw=True, max_bytes=1024 * 1024)
lines = redact_logs(decode_docker_output(data)).splitlines()
if query:
lines = [line for line in lines if query.casefold() in line.casefold()]
return {"node": name, "lines": lines[-1000:], "log_policy": detail.get("HostConfig", {}).get("LogConfig", {}), "limited_bytes": 1024 * 1024}
async def archive_logs(self, engine, name):
"""Keep a bounded redacted tail BEFORE removing an owned container."""
import gzip
encoded = quote(name, safe="")
params = urlencode({"stdout": 1, "stderr": 1, "timestamps": 1, "tail": 10000})
data = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/logs?{params}", raw=True, max_bytes=2 * 1024 * 1024)
archive_id = f"{name}-{time.time_ns()}"
payload = {"node": name, "created_at": time.time(), "lines": redact_logs(decode_docker_output(data)).splitlines(), "limited_bytes": 2 * 1024 * 1024, "note": "节点删除前的最近 10000 行 / 最多 2 MB 归档;保存 7 天,最多 50 份。"}
self.archive_directory.mkdir(mode=0o700, parents=True, exist_ok=True)
target = self.archive_directory / f"{archive_id}.json.gz"
fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "wb") as out:
out.write(gzip.compress(json.dumps(payload).encode()))
files = sorted(self.archive_directory.glob("*.json.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
for index, path in enumerate(files):
if index >= 50 or path.stat().st_mtime < time.time() - 7 * 86400:
path.unlink()
def archives(self):
return [
{"id": p.name.removesuffix(".json.gz"), "name": p.name.removesuffix(".json.gz"), "created_at": p.stat().st_mtime}
for p in sorted(self.archive_directory.glob("*.json.gz"), key=lambda p: p.stat().st_mtime, reverse=True)[:50]
if p.stat().st_mtime >= time.time() - 7 * 86400
]
def read_archive(self, archive_id, query=""):
import gzip
if not re.fullmatch(r"[A-Za-z0-9_.-]{1,180}", archive_id):
raise DockerError("无效日志归档")
path = self.archive_directory / f"{archive_id}.json.gz"
if not path.is_file() or path.stat().st_mtime < time.time() - 7 * 86400:
raise DockerError("日志归档不存在或已过期")
payload = json.loads(gzip.decompress(path.read_bytes()))
payload["lines"] = [line for line in payload["lines"] if query.casefold() in line.casefold()][-1000:]
return payload
@@ -0,0 +1,60 @@
"""Read-only server discovery and conservative starter recommendations."""
import base64
import hashlib
import math
import socket
from .docker import DockerEngine, DockerError
def host_fingerprint(host: str, port: int) -> str:
import paramiko
transport = None
try:
with socket.create_connection((host, port), timeout=10) as connection:
transport = paramiko.Transport(connection)
transport.start_client(timeout=10)
key = transport.get_remote_server_key()
return "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
except Exception as exc:
raise DockerError(f"读取服务器指纹失败({type(exc).__name__})") from None
finally:
if transport is not None:
transport.close()
def recommend(cpus, memory_mb, reserved_cpu=0, reserved_memory_mb=0, connection_budget=64):
# Reserve at least half the host for the main app, DB and other services.
cpu_budget = max(0, min(cpus / 2, cpus - reserved_cpu - 1))
memory_budget = max(0, min(memory_mb // 2, memory_mb - reserved_memory_mb - 2048))
cpu = 2 if cpu_budget >= 2 else 1
memory = 2048
count = max(0, min(4, math.floor(cpu_budget / cpu), memory_budget // memory, connection_budget // 12))
return {"desired": int(count), "cpu": cpu, "memory_mb": memory,
"total_cpu": max(cpu, count * cpu), "total_memory_mb": max(memory, count * memory),
"concurrency": 4, "database_pool_size": 4, "postgres_pool_size": 4}
def discover(settings, instance):
engine = DockerEngine(settings)
info = engine.request("GET", "/info")
containers = engine.request("GET", "/containers/json")
cpu_reserved = 0
memory_reserved = 0
templates = []
for item in containers:
if item.get("Labels", {}).get("deerflow.worker-pool") == instance:
continue
detail = engine.request("GET", f"/containers/{item['Id']}/json")
limits = detail.get("HostConfig", {})
cpu_reserved += limits.get("NanoCpus", 0) / 1_000_000_000
memory_reserved += limits.get("Memory", 0) // (1024 ** 2)
if item.get("Labels", {}).get("deerflow.worker-role") == "online":
continue
networks = [name for name in detail.get("NetworkSettings", {}).get("Networks", {}) if name not in {"host", "none", "bridge"}]
templates.append({"name": detail.get("Name", "").lstrip("/"), "networks": networks})
cpus = int(info.get("NCPU", 0))
memory_mb = int(info.get("MemTotal", 0)) // (1024 ** 2)
return {"cpus": cpus, "memory_mb": memory_mb, "containers": templates,
"recommendation": recommend(cpus, memory_mb, cpu_reserved, memory_reserved, settings.database_connection_budget),
"note": "按硬件总量和其他容器已配置的资源上限估算,至少为主服务、数据库及系统保留一半资源;不是实时空闲内存。建议先用最多 4 个节点,再按实际负载调整。"}
@@ -0,0 +1,505 @@
"""Docker Engine API over a Unix socket or a pinned SSH connection.
Only the constant `docker system dial-stdio` command runs on the server.
All variable data travels as JSON, never interpolated into a shell command.
"""
from __future__ import annotations
import base64
import hashlib
import http.client
import json
import posixpath
import socket
from urllib.parse import quote
from .settings import PoolSettings
class DockerError(RuntimeError):
pass
_WORKER_STORAGE_SELECTION = """\
import os
from pathlib import Path
import yaml
config_path = Path(os.getenv('DEER_FLOW_CONFIG_PATH') or 'config.yaml')
config = yaml.safe_load(config_path.read_text(encoding='utf-8')) or {}
checkpoint = config.get('checkpointer')
if checkpoint is not None:
checkpoint_type = checkpoint.get('type')
if checkpoint_type == 'postgres':
raw = os.getenv('DEER_FLOW_USE_PG_CHECKPOINTER')
if raw:
postgres_enabled = raw.strip().lower() not in {'0', 'false', 'no', 'off'}
else:
postgres_enabled = bool(checkpoint.get('use_postgres', True))
backend = 'postgres' if postgres_enabled else 'sqlite'
elif checkpoint_type == 'sqlite':
backend = 'sqlite'
else:
backend = 'memory'
else:
backend = (config.get('database') or {}).get('backend', 'memory')
if backend not in {'postgres', 'sqlite'}:
backend = 'memory'
"""
_WORKER_STORAGE_PROBE = _WORKER_STORAGE_SELECTION + """\
print('__DEERFLOW_STORAGE_BACKEND__=' + backend)
"""
_WORKER_PYTHON_PROBE = _WORKER_STORAGE_SELECTION + """\
import importlib.util
import os
import sys
if sys.version_info < (3, 12):
raise SystemExit(78)
required = ['uvicorn']
if backend == 'postgres':
required.extend((
'langgraph.checkpoint.postgres.aio',
'langgraph.store.postgres.aio',
'psycopg.rows',
'psycopg_pool',
))
elif backend == 'sqlite':
required.extend((
'langgraph.checkpoint.sqlite.aio',
'langgraph.store.sqlite.aio',
'aiosqlite',
))
missing = []
for module in required:
try:
if importlib.util.find_spec(module) is None:
missing.append(module)
except (ImportError, ModuleNotFoundError, ValueError):
missing.append(module)
if not Path('app/gateway/worker_pool/worker.py').is_file():
missing.append('app.gateway.worker_pool.worker')
if missing:
packages = (
'langgraph-checkpoint-postgres psycopg[binary] psycopg-pool'
if backend == 'postgres'
else 'langgraph-checkpoint-sqlite aiosqlite'
)
print('__DEERFLOW_STORAGE_ERROR__=Worker Python is missing configured runtime modules: '
+ ', '.join(missing) + '; install ' + packages, flush=True)
raise SystemExit(78)
print('__DEERFLOW_WORKER_PYTHON__=' + os.path.abspath(sys.executable))
"""
_WORKER_PYTHON_PROBE_SHELL = r'''
probe() {
candidate="$1"
[ -n "$candidate" ] || return 1
output=$("$candidate" -c "$2" 2>/dev/null) || {
case "$output" in
*__DEERFLOW_STORAGE_ERROR__=*)
error=${output##*__DEERFLOW_STORAGE_ERROR__=}
printf '__DEERFLOW_STORAGE_ERROR__=%s\n' "$error"
;;
esac
return 1
}
case "$output" in
*__DEERFLOW_WORKER_PYTHON__=*)
resolved=${output##*__DEERFLOW_WORKER_PYTHON__=}
case "$resolved" in /*) printf '%s\n' "$resolved"; return 0 ;; esac
;;
esac
return 1
}
code="$1"
shift
for candidate in "$@"; do
if probe "$candidate" "$code"; then exit 0; fi
done
# The main API may use a Python executable outside the image's default PATH.
# Probe the interpreter actually running in this source container before PATH
# fallbacks, using the same configured persistence-adapter check above.
for process_exe in /proc/[0-9]*/exe; do
candidate=$(readlink "$process_exe" 2>/dev/null) || continue
case "$candidate" in
*/python|*/python[0-9]|*/python[0-9].* )
case "$candidate" in *" (deleted)"*) continue ;; esac
if probe "$candidate" "$code"; then exit 0; fi
;;
esac
done
for candidate in python3 python3.14 python3.13 python3.12 python3.11 python; do
if probe "$candidate" "$code"; then exit 0; fi
done
exit 1
'''
class DockerEngine:
def __init__(self, settings: PoolSettings):
self.settings = settings
def request(self, method: str, path: str, body: dict | None = None, *, raw=False, max_bytes=None, json_stream=False):
client = None
channel = None
# Committing a large production backend layer can take longer than
# the ordinary metadata/exec requests. Do not turn a slow but valid
# runtime snapshot into a permanent Worker reconciliation failure.
request_timeout = 300 if method == "POST" and path.startswith("/commit?") else 20
try:
if self.settings.transport == "socket":
channel = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
channel.settimeout(request_timeout)
channel.connect("/var/run/docker.sock")
else:
import paramiko
expected = self.settings.host_key_sha256
if not expected.startswith("SHA256:"):
raise DockerError("请先填写服务器 SSH 主机密钥指纹(SHA256:…)")
class PinnedKey(paramiko.MissingHostKeyPolicy):
def missing_host_key(self, ssh, hostname, key):
actual = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
if actual != expected:
raise DockerError("SSH 主机密钥指纹不匹配,连接已拒绝")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(PinnedKey())
client.connect(
self.settings.host,
port=self.settings.port,
username=self.settings.username,
password=self.settings.password,
timeout=10,
auth_timeout=10,
banner_timeout=10,
look_for_keys=False,
allow_agent=False,
)
channel = client.get_transport().open_session(timeout=20)
channel.settimeout(request_timeout)
channel.exec_command("docker system dial-stdio")
data = json.dumps(body).encode() if body is not None else b""
request = f"{method} {path} HTTP/1.1\r\nHost: docker\r\nContent-Type: application/json\r\nContent-Length: {len(data)}\r\nConnection: close\r\n\r\n".encode() + data
channel.sendall(request)
response = http.client.HTTPResponse(channel)
response.begin()
if json_stream and response.status < 400:
# Exec uses a hijacked connection that may outlive the process.
# Stop on a complete bounded JSON result instead of waiting for EOF.
result = b""
while len(result) < (max_bytes or 1024 * 1024):
chunk = response.read1(min(65536, (max_bytes or 1024 * 1024) - len(result)))
if not chunk:
break
result += chunk
try:
json.loads(decode_docker_output(result))
break
except ValueError:
pass
else:
result = response.read(max_bytes) if max_bytes else response.read()
if response.status >= 400:
# Docker errors can contain environment values / host paths.
raise DockerError(f"Docker 操作失败(HTTP {response.status}),请检查容器、网络和账号权限")
return result if raw else (json.loads(result) if result else {})
except DockerError:
raise
except Exception as exc:
raise DockerError(f"服务器连接失败({type(exc).__name__}),请检查地址、账号、密码和 Docker 权限") from None
finally:
if channel is not None:
channel.close()
if client is not None:
client.close()
def _exec_capture(self, container_id: str, command: list[str], working_dir: str, user: str) -> tuple[int, str]:
created = self.request("POST", f"/containers/{quote(container_id, safe='')}/exec", {
"AttachStdout": True,
"AttachStderr": True,
"Cmd": command,
"WorkingDir": working_dir,
"User": user,
})
exec_id = quote(created["Id"], safe="")
output = self.request(
"POST", f"/exec/{exec_id}/start", {"Detach": False, "Tty": False},
raw=True, max_bytes=1024 * 1024,
)
inspected = self.request("GET", f"/exec/{exec_id}/json")
exit_code = inspected.get("ExitCode")
return (int(exit_code) if exit_code is not None else 1), decode_docker_output(output)
@staticmethod
def _python_candidates(source: dict, preferred: str) -> list[str]:
candidates = [
preferred,
"/opt/venv/bin/python", "/opt/venv/bin/python3",
"/app/.venv/bin/python", "/app/.venv/bin/python3",
"/app/backend/.venv/bin/python", "/venv/bin/python", "/venv/bin/python3",
"/usr/local/bin/python", "/usr/local/bin/python3",
"/usr/local/bin/python3.14", "/usr/local/bin/python3.13",
"/usr/local/bin/python3.12", "/usr/local/bin/python3.11",
"/usr/bin/python", "/usr/bin/python3", "/usr/bin/python3.12",
]
env = source.get("Config", {}).get("Env", [])
path_value = next((item[5:] for item in env if item.startswith("PATH=")), "")
for directory in path_value.split(":"):
if directory.startswith("/"):
candidates.extend(f"{directory.rstrip('/')}/{name}" for name in ("python3", "python"))
return list(dict.fromkeys(path for path in candidates if path and path.startswith("/")))
def discover_worker_python(self, source: dict, preferred: str, working_dir: str) -> str:
"""Find an interpreter inside the source image that can actually start a Worker."""
container_id = source.get("Id")
if not container_id:
raise DockerError("无法读取主后端容器 ID,不能检测镜像内 Python")
config = source.get("Config", {})
user = config.get("User", "")
candidates = self._python_candidates(source, preferred)
storage_error = None
try:
code, output = self._exec_capture(
container_id,
["/bin/sh", "-c", _WORKER_PYTHON_PROBE_SHELL, "worker-python-probe", _WORKER_PYTHON_PROBE, *candidates],
working_dir,
user,
)
if code == 0:
for line in reversed(output.splitlines()):
if line.startswith("/") and "\x00" not in line:
return line.strip()
for line in output.splitlines():
prefix = "__DEERFLOW_STORAGE_ERROR__="
if line.startswith(prefix):
storage_error = line[len(prefix):].strip()
except DockerError:
# Minimal images may not include a shell; try known Python paths directly.
pass
for candidate in candidates:
try:
code, output = self._exec_capture(
container_id,
[candidate, "-c", _WORKER_PYTHON_PROBE],
working_dir,
user,
)
except DockerError:
continue
if code == 0:
for line in reversed(output.splitlines()):
prefix = "__DEERFLOW_WORKER_PYTHON__="
if line.startswith(prefix):
resolved = line[len(prefix):].strip()
if resolved.startswith("/") and "\x00" not in resolved:
return resolved
for line in output.splitlines():
prefix = "__DEERFLOW_STORAGE_ERROR__="
if line.startswith(prefix):
storage_error = line[len(prefix):].strip()
if storage_error:
raise DockerError(storage_error)
raise DockerError(
"未找到与主服务聊天存储配置兼容的 Worker Python(要求 Python 3.12+,"
"且能导入 uvicorn、Worker 入口及当前 SQLite/PostgreSQL 适配器);"
"已检查主服务常见 Python 路径和运行进程解释器,请确认 Worker 复用了主服务的镜像、"
"环境变量、挂载目录和 Python 环境"
)
def configured_storage_backend(self, source: dict, preferred: str, working_dir: str) -> str:
"""Read the live config so SQLite→PostgreSQL changes invalidate probes."""
container_id = source.get("Id")
if not container_id:
raise DockerError("无法读取主后端容器 ID,不能检查聊天存储配置")
user = source.get("Config", {}).get("User", "")
for candidate in self._python_candidates(source, preferred):
try:
code, output = self._exec_capture(
container_id,
[candidate, "-c", _WORKER_STORAGE_PROBE],
working_dir,
user,
)
except DockerError:
continue
if code == 0:
for line in reversed(output.splitlines()):
prefix = "__DEERFLOW_STORAGE_BACKEND__="
if line.startswith(prefix):
backend = line[len(prefix):].strip()
if backend in {"postgres", "sqlite", "memory"}:
return backend
raise DockerError(
"无法读取主服务当前聊天存储配置;请确认 Python 路径、工作目录和主服务配置文件正确"
)
def snapshot_worker_runtime(self, source: dict, instance: str) -> str:
"""Snapshot the running main container's writable layer for new workers.
Docker create from Config.Image alone loses packages installed into the
running container after it was created. A local commit carries those
offline-installed packages; bind/volume mounts are still cloned as-is.
"""
container_id = source.get("Id")
if not container_id:
raise DockerError("无法读取主后端容器 ID,不能复用其 Python 运行环境")
# A freshly started backend often has only a tiny writable overlay
# (logs/runtime markers). Committing it can take minutes on an offline
# Docker host even though the immutable image already has the runtime.
# Keep committing larger overlays that may include offline-installed
# Python packages.
base_image = source.get("Image")
writable_layer_size = source.get("SizeRw")
if (
isinstance(base_image, str)
and base_image
and isinstance(writable_layer_size, int)
and writable_layer_size <= 128 * 1024
):
return base_image
repo = f"deerflow-worker-runtime-{instance}"
tag = container_id[:12]
result = self.request(
"POST",
f"/commit?container={quote(container_id, safe='')}&repo={quote(repo, safe='')}&tag={quote(tag, safe='')}&pause=false",
)
image_id = result.get("Id") if isinstance(result, dict) else None
if not isinstance(image_id, str) or not image_id.startswith("sha256:"):
raise DockerError("无法从主容器生成 Worker 运行镜像;请确认 Docker 账号具有容器提交权限")
return image_id
def host_snapshot(self, container: str):
"""Only our constant read-only collector may be executed by diagnostics."""
if self.settings.transport == "socket":
from .host_metrics import read_snapshot
return read_snapshot()
command = [self.settings.python_executable, "-m", "app.gateway.worker_pool.host_metrics"]
created = self.request("POST", f"/containers/{quote(container, safe='')}/exec", {
"AttachStdout": True, "AttachStderr": True, "Cmd": command,
"WorkingDir": self.settings.working_dir,
})
data = self.request("POST", f"/exec/{created['Id']}/start", {"Detach": False, "Tty": False}, raw=True, max_bytes=1024 * 1024, json_stream=True)
out = decode_docker_output(data)
try:
return json.loads(out)
except ValueError:
raise DockerError("宿主机采集器未运行成功,请检查主容器代码和只读 /proc、/sys 挂载") from None
def decode_docker_output(data: bytes) -> str:
"""Docker attaches stdout/stderr in 8-byte length-prefixed frames."""
import struct
if len(data) < 8 or data[0] not in (0, 1, 2) or data[1:4] != b"\x00\x00\x00":
return data.decode("utf-8", errors="replace")
parts = []
offset = 0
while offset + 8 <= len(data):
size = struct.unpack(">I", data[offset + 4:offset + 8])[0]
parts.append(data[offset + 8:offset + 8 + size])
offset += 8 + size
return b"".join(parts).decode("utf-8", errors="replace")
def worker_spec(
source: dict,
settings: PoolSettings,
instance: str,
token: str,
control_home: str,
*,
runtime_image: str | None = None,
require_shared_archive: bool = False,
) -> dict:
"""Clone an explicit runtime template, without ports, privileges or Docker access."""
if settings.network not in source["NetworkSettings"]["Networks"]:
raise DockerError("源容器必须连接所选 Docker 网络")
if settings.network in {"host", "none", "bridge"}:
raise DockerError("请选择用户自定义 Docker 网络")
mounts = []
for mount in source.get("Mounts", []):
target = mount["Destination"].rstrip("/")
if control_home == target:
continue # dedicated controller volume is never passed to workers
if "docker.sock" in target or target in {"/", "/var/run", "/run"}:
continue
if target in {"/host/proc", "/host/sys"}:
continue # host diagnostics belong to the controller only
if control_home.startswith(target + "/"):
raise DockerError("控制器密钥目录不能与 Worker 共享,请为 DEER_FLOW_WORKER_CONTROL_HOME 配置独立持久卷")
if mount["Type"] not in {"bind", "volume"}:
continue
mounts.append({"Type": mount["Type"], "Source": mount.get("Name") if mount["Type"] == "volume" else mount["Source"], "Target": mount["Destination"], "ReadOnly": not mount.get("RW", True)})
env = dict(item.split("=", 1) for item in source["Config"].get("Env", []) if "=" in item)
if not env.get("AUTH_JWT_SECRET"):
raise DockerError("请在源容器环境变量中设置固定 AUTH_JWT_SECRET,使 Worker 能验证现有登录会话")
if not mounts:
raise DockerError("源容器没有共享持久化挂载,无法保证 Worker 访问技能、配置和上传文件")
if require_shared_archive:
if env.get("DEER_FLOW_CHAT_ARCHIVE_DIR"):
archive_path = posixpath.normpath(env["DEER_FLOW_CHAT_ARCHIVE_DIR"])
elif env.get("DEER_FLOW_HOME"):
archive_path = posixpath.join(env["DEER_FLOW_HOME"], "data", "chat-archives")
else:
project_root = env.get("DEER_FLOW_PROJECT_ROOT") or source["Config"].get("WorkingDir") or settings.working_dir
archive_path = posixpath.join(project_root, ".deer-flow", "data", "chat-archives")
if not posixpath.isabs(archive_path):
workdir = source["Config"].get("WorkingDir") or settings.working_dir
archive_path = posixpath.join(workdir, archive_path)
archive_path = posixpath.normpath(archive_path)
shared = any(
mount.get("ReadOnly") is False
and (archive_path == mount["Target"].rstrip("/")
or archive_path.startswith(mount["Target"].rstrip("/") + "/"))
for mount in mounts
)
if not shared:
raise DockerError(
f"PostgreSQL 聊天归档目录 {archive_path} 必须位于主容器与 Worker 共享的可写持久卷中;"
"请将 DEER_FLOW_CHAT_ARCHIVE_DIR 指向共享卷目录或挂载 DEER_FLOW_HOME/data"
)
for key in list(env):
if key.startswith("DEER_FLOW_WORKER_") or key in {"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA", "WEB_CONCURRENCY", "UVICORN_WORKERS"}:
env.pop(key)
env.update(
DEER_FLOW_WORKER_DB_POOL_SIZE=str(settings.database_pool_size),
DEER_FLOW_PG_POOL_MIN="1",
DEER_FLOW_PG_POOL_MAX=str(settings.postgres_pool_size),
DEER_FLOW_WORKER_ROLE="online",
DEER_FLOW_WORKER_TOKEN=token,
DEER_FLOW_WORKER_CONCURRENCY=str(settings.concurrency),
DEER_FLOW_WORKER_QUEUE_LIMIT=str(settings.queue_limit),
DEER_FLOW_WORKER_QUEUE_TIMEOUT=str(settings.queue_timeout),
SCHEDULED_TASKS_ENABLED="false",
TASK_ANALYSIS_ENABLED="false",
)
return {
"Image": runtime_image or source["Image"],
"WorkingDir": settings.working_dir,
"User": source["Config"].get("User", ""),
"Entrypoint": [settings.python_executable, "-m", "uvicorn"],
"Cmd": ["app.gateway.worker_pool.worker:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"],
"Env": [f"{key}={value}" for key, value in env.items()],
"Labels": {"deerflow.worker-pool": instance, "deerflow.worker-role": "online"},
"NetworkingConfig": {"EndpointsConfig": {name: {} for name in source["NetworkSettings"]["Networks"] if name not in {"host", "none", "bridge"}}},
"Healthcheck": {
"Test": ["CMD", settings.python_executable, "-c", "import os,urllib.request; urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:8001/internal/worker-status',headers={'x-deerflow-worker-token':os.environ['DEER_FLOW_WORKER_TOKEN']}),timeout=4).read()"],
"Interval": 20_000_000_000, "Timeout": 5_000_000_000, "Retries": 6, "StartPeriod": 90_000_000_000,
},
"HostConfig": {
"Mounts": mounts,
"NetworkMode": settings.network,
"Memory": settings.memory_mb * 1024 * 1024,
"NanoCpus": int(settings.cpu * 1_000_000_000),
"PidsLimit": 512,
"RestartPolicy": {"Name": "unless-stopped"},
"SecurityOpt": ["no-new-privileges:true"],
"ExtraHosts": source.get("HostConfig", {}).get("ExtraHosts") or [],
"Dns": source.get("HostConfig", {}).get("Dns") or [],
"Init": True,
"LogConfig": {"Type": "json-file", "Config": {"max-size": "10m", "max-file": "3"}},
},
}
@@ -0,0 +1,110 @@
"""Small read-only host snapshot; requires /proc and /sys mounted by the admin."""
import json
import os
import re
import time
from pathlib import Path
def read_snapshot(proc=Path("/host/proc"), sys=Path("/host/sys")):
if not (proc / "stat").is_file():
return {"available": False, "message": "请为主容器挂载 /proc:/host/proc:ro 和 /sys:/host/sys:ro;Worker 不需要挂载。"}
cpu = next(line for line in (proc / "stat").read_text().splitlines() if line.startswith("cpu "))
fields = [int(v) for v in cpu.split()[1:9]] # guest times are already included in user/nice
mem = {}
for line in (proc / "meminfo").read_text().splitlines():
k, value = line.split(":", 1)
mem[k] = int(value.split()[0]) * 1024
disks = {}
for line in (proc / "diskstats").read_text().splitlines():
v = line.split()
if len(v) < 14 or re.match(r"^(loop|ram|fd|sr)", v[2]):
continue
# Whole devices only: including partitions would double-count disk IO.
if (sys / "class/block" / v[2] / "partition").exists():
continue
disks[v[2]] = {"read_bytes": int(v[5]) * 512, "write_bytes": int(v[9]) * 512,
"io_ms": int(v[12]), "inflight": int(v[11])}
interfaces = {}
network_error = None
try:
# /proc/1 is the host PID namespace; /proc/net would resolve to this container.
for line in (proc / "1/net/dev").read_text().splitlines()[2:]:
name, values = line.split(":", 1)
name = name.strip()
if name == "lo" or name.startswith(("veth", "docker", "br-", "virbr", "tun", "tap")):
continue
v = values.split()
speed = None
try:
speed = int((sys / "class/net" / name / "speed").read_text())
except (OSError, ValueError):
pass
interfaces[name] = {"rx_bytes": int(v[0]), "tx_bytes": int(v[8]),
"rx_errors": int(v[2]), "tx_errors": int(v[10]), "link_mbps": speed if speed and speed > 0 else None}
except OSError:
network_error = "无法读取宿主机网络命名空间,检查 /host/proc 挂载及读取权限"
pressure = {}
for name in ("cpu", "memory", "io"):
try:
pressure[name] = (proc / "pressure" / name).read_text().strip()[:400]
except OSError:
pressure[name] = None
processes = {}
page_size = getattr(os, "sysconf", lambda _: 4096)("SC_PAGE_SIZE")
for item in proc.iterdir():
if not item.name.isdigit():
continue
if len(processes) >= 5000:
break
try:
stat = (item / "stat").read_text()
end = stat.rfind(")")
process_fields = stat[end + 2:].split()
processes[item.name] = {"name": stat[stat.find("(") + 1:end][:80], "ticks": int(process_fields[11]) + int(process_fields[12]),
"started": int(process_fields[19]), "rss_bytes": max(0, int(process_fields[21])) * page_size}
except (OSError, ValueError, IndexError):
continue
return {"available": True, "at": time.time(), "cpu_total": sum(fields), "cpu_idle": fields[3] + fields[4],
"cpu_iowait": fields[4], "cpu_count": sum(line.startswith("cpu") and line[3:4].isdigit() for line in (proc / "stat").read_text().splitlines()),
"memory_total": mem.get("MemTotal", 0), "memory_available": mem.get("MemAvailable", mem.get("MemFree", 0)),
"swap_total": mem.get("SwapTotal", 0), "swap_free": mem.get("SwapFree", 0),
"load": [float(v) for v in (proc / "loadavg").read_text().split()[:3]],
"disks": disks, "interfaces": interfaces, "network_error": network_error, "pressure": pressure,
"process_counters": processes, "clock_ticks": getattr(os, "sysconf", lambda _: 100)("SC_CLK_TCK")}
def rates(current, previous):
result = dict(current)
if not current.get("available"):
return result
result.update(cpu_percent=None, iowait_percent=None)
elapsed = current["at"] - previous["at"] if previous and previous.get("available") else 0
if elapsed > 0:
delta = current["cpu_total"] - previous["cpu_total"]
if delta > 0:
result["cpu_percent"] = round(100 * (1 - (current["cpu_idle"] - previous["cpu_idle"]) / delta), 2)
result["iowait_percent"] = round(100 * (current["cpu_iowait"] - previous["cpu_iowait"]) / delta, 2)
top = []
for pid, item in current.get("process_counters", {}).items():
old = previous.get("process_counters", {}).get(pid) if elapsed > 0 else None
if old and old["started"] == item["started"]:
percent = max(0, (item["ticks"] - old["ticks"]) / (current.get("clock_ticks", 100) * elapsed) * 100)
top.append({"pid": int(pid), "name": item["name"], "cpu_percent": round(percent, 2), "rss_bytes": item["rss_bytes"]})
result.pop("process_counters", None)
result["top_processes"] = sorted(top, key=lambda p: p["cpu_percent"], reverse=True)[:20]
for name, item in result["disks"].items():
old = previous.get("disks", {}).get(name) if elapsed > 0 else None
item.update(read_bps=max(0, (item["read_bytes"] - old["read_bytes"]) / elapsed) if old else None,
write_bps=max(0, (item["write_bytes"] - old["write_bytes"]) / elapsed) if old else None,
busy_percent=min(100, max(0, (item["io_ms"] - old["io_ms"]) / (elapsed * 10))) if old else None)
for name, item in result["interfaces"].items():
old = previous.get("interfaces", {}).get(name) if elapsed > 0 else None
item.update(rx_bps=max(0, (item["rx_bytes"] - old["rx_bytes"]) / elapsed) if old else None,
tx_bps=max(0, (item["tx_bytes"] - old["tx_bytes"]) / elapsed) if old else None)
return result
if __name__ == "__main__":
print(json.dumps(read_snapshot()))
@@ -0,0 +1,51 @@
"""Create the opt-in Worker controller without restricting normal startup."""
import json
import os
from contextlib import asynccontextmanager
from pathlib import Path
def _control_directory() -> Path:
return Path(os.getenv("DEER_FLOW_WORKER_CONTROL_HOME", ".worker-pool")).resolve()
def _persisted_feature_enabled(directory: Path) -> bool:
def read(path):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
state = read(directory / "settings.json")
diagnostics = read(directory / "diagnostics-settings.json")
return bool(state.get("settings", {}).get("enabled") or diagnostics.get("enabled"))
def ensure_controller(app):
pool = getattr(app.state, "worker_pool", None)
if pool is not None:
return pool
from .controller import Controller
from .diagnostics import Diagnostics
from .settings import SettingsStore
directory = getattr(app.state, "worker_pool_directory", _control_directory())
pool = Controller(SettingsStore(directory))
pool.diagnostics = Diagnostics(pool, app)
app.state.worker_pool = pool
return pool
@asynccontextmanager
async def managed_controller(app):
directory = _control_directory()
app.state.worker_pool_directory = directory
app.state.worker_pool = None
pool = ensure_controller(app) if _persisted_feature_enabled(directory) else None
try:
yield pool
finally:
pool = getattr(app.state, "worker_pool", None)
if pool is not None:
await pool.close()
@@ -0,0 +1,31 @@
"""A pinned online worker can identify its own orphaned in-flight records."""
import inspect
from deerflow.runtime.runs.manager import RunManager
from deerflow.runtime.runs.schemas import RunStatus
class WorkerRunManager(RunManager):
def has_active_thread(self, thread_id: str) -> bool:
return any(
record.thread_id == thread_id
and record.status in {RunStatus.pending, RunStatus.running}
and (record.task is None or not record.task.done())
for record in self._runs.values()
)
async def get_persisted(self, run_id: str, *, authorized_thread_id: str | None = None):
if "authorized_thread_id" in inspect.signature(RunManager.get_persisted).parameters:
record = await super().get_persisted(run_id, authorized_thread_id=authorized_thread_id)
else:
record = await super().get_persisted(run_id)
if record is not None and authorized_thread_id and record.thread_id != authorized_thread_id:
return None
if record is not None and self.get(run_id) is None and record.status in {RunStatus.pending, RunStatus.running}:
# Routing never migrates a live process. Missing live state means a
# previous process ended; replaying tools here would be unsafe.
record.status = RunStatus.error
record.error = "执行节点已重启,本轮已中断;已保存的聊天记录保留。请核对外部操作结果后再继续。"
await self._store.update_status(run_id, record.status.value, error=record.error)
return record
@@ -0,0 +1,68 @@
"""Match the runtime's legacy-first persistence configuration precedence."""
import os
from pathlib import Path
def postgres_connection_string(config) -> str | None:
if not shared_postgres_configured(config):
return None
checkpoint = getattr(config, "checkpointer", None)
if checkpoint is not None:
return getattr(checkpoint, "connection_string", None)
return getattr(config.database, "postgres_url", None)
def shared_postgres_configured(config) -> bool:
database = getattr(config, "database", None)
if getattr(database, "backend", None) not in {"postgres", "mysql"}:
return False
checkpoint = getattr(config, "checkpointer", None)
if checkpoint is None:
return database.backend == "postgres"
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
enabled = raw.strip().lower() not in {"0", "false", "no", "off"} if raw else getattr(checkpoint, "use_postgres", True)
return checkpoint.type == "postgres" and enabled
def sqlite_checkpoint_path(config) -> str | None:
from deerflow.runtime.checkpointer.async_provider import _sqlite_conn_str_for_postgres_fallback, _use_postgres_enabled
from deerflow.runtime.store._sqlite_utils import resolve_sqlite_conn_str
checkpoint = getattr(config, "checkpointer", None)
if checkpoint is None:
return None
if checkpoint.type == "sqlite":
raw = getattr(checkpoint, "connection_string", None) or "store.db"
elif checkpoint.type == "postgres" and not _use_postgres_enabled(checkpoint):
raw = _sqlite_conn_str_for_postgres_fallback(checkpoint)
else:
return None
return resolve_sqlite_conn_str(raw)
def shared_storage_configured(config) -> bool:
if shared_postgres_configured(config):
return True
if getattr(getattr(config, "database", None), "backend", None) not in {"mysql", "postgres"}:
return False
path = sqlite_checkpoint_path(config)
return bool(path and path != ":memory:" and not path.startswith("file:"))
def validate_sqlite_mount(config, mountinfo: str | None = None) -> None:
"""SQLite databases and WAL sidecars must live on one shared local directory."""
path = sqlite_checkpoint_path(config)
if path is None:
return
database = Path(path).resolve()
content = mountinfo if mountinfo is not None else Path("/proc/self/mountinfo").read_text()
for line in content.splitlines():
left, right = line.split(" - ", 1)
mount = Path(left.split()[4].replace("\\040", " "))
# A single-file bind does not share the -wal/-shm files.
if mount != Path("/") and mount in database.parents:
if right.split()[0] in {"nfs", "nfs4", "cifs", "smb3", "9p"}:
raise RuntimeError("SQLite Worker requires a shared local directory, not a network filesystem")
return
raise RuntimeError("SQLite Worker requires the database directory to be persistently mounted in every container")
@@ -0,0 +1,153 @@
"""ASGI streaming proxy. Preserve authentication, disconnects and SSE headers."""
from __future__ import annotations
import asyncio
import logging
import re
import time
from urllib.parse import quote
import httpx
from starlette.requests import Request
from starlette.responses import JSONResponse, StreamingResponse
from .controller import PoolUnavailable
THREAD_PATH = re.compile(r"^/api/threads/([^/]+)(?:/.*)?$")
START_PATH = re.compile(r"^/api/threads/[^/]+/runs(?:/(?:stream|wait))?$")
HOP_HEADERS = {b"host", b"connection", b"keep-alive", b"transfer-encoding", b"upgrade", b"proxy-authenticate", b"proxy-authorization", b"te", b"trailer", b"content-length", b"x-deerflow-worker-token"}
def _main_has_persistent_chat_storage(app) -> bool:
checkpointer = getattr(app.state, "checkpointer", None)
store = getattr(app.state, "store", None)
if checkpointer is None or store is None:
return False
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.store.memory import InMemoryStore
return not isinstance(checkpointer, InMemorySaver) and not isinstance(store, InMemoryStore)
async def send_upstream(client, method, url, headers, content):
# A keep-alive socket may close while a completed run is being queried.
# Retry only body-free reads, before downstream headers have been sent.
attempts = 2 if method in {"GET", "HEAD"} else 1
for attempt in range(attempts):
upstream = client.build_request(method, url, headers=headers, content=None if method in {"GET", "HEAD"} else content)
try:
return await client.send(upstream, stream=True)
except httpx.TransportError:
if attempt + 1 == attempts:
raise
await asyncio.sleep(0.05)
class WorkerProxy:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
pool = getattr(scope["app"].state, "worker_pool", None)
match = THREAD_PATH.fullmatch(scope["path"])
if pool is None or match is None or match[1] in {"search", "count", "reference-debug-log"}:
return await self.app(scope, receive, send)
if pool.db is None and not pool.store.load().enabled:
return await self.app(scope, receive, send)
row = pool._ensure_db().execute("SELECT node FROM affinity WHERE thread=?", (match[1],)).fetchone()
creating = scope["method"] == "POST" and bool(START_PATH.fullmatch(scope["path"]))
if row is None and (not creating or not pool.store.load().enabled):
return await self.app(scope, receive, send)
# Authenticate before allocating durable routing state. The worker
# still performs the existing owner/permission checks for every route.
request = Request(scope, receive)
from fastapi import HTTPException
from app.gateway.deps import get_current_user_from_request, get_thread_store
try:
access_started = time.perf_counter()
user = getattr(request.state, "user", None) or await get_current_user_from_request(request)
allowed = await get_thread_store(request).check_access(match[1], str(user.id), require_existing=True)
if not allowed:
return await JSONResponse({"detail": "Thread not found"}, 404)(scope, receive, send)
if row is None:
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
record = await get_thread_store(request).get(match[1], user_id=None)
if record and is_excluded_system_thread(record.get("metadata")):
return await self.app(scope, receive, send)
manager = getattr(scope["app"].state, "run_manager", None)
if creating and pool.local_threads.get(match[1], 0):
return await JSONResponse({"detail": "会话仍在主服务执行,请完成当前轮后再提交"}, 409)(scope, receive, send)
if creating and manager is not None and any(
record.thread_id == match[1]
and getattr(record.status, "value", record.status) in {"pending", "running"}
and (record.task is None or not record.task.done())
for record in manager._runs.values()
):
return await JSONResponse({"detail": "会话仍在执行,请完成当前轮后再提交"}, 409)(scope, receive, send)
node = await pool.reserve(match[1], create=creating)
scope.setdefault("state", {})["worker_access_ms"] = round((time.perf_counter() - access_started) * 1000, 2)
except HTTPException as exc:
return await JSONResponse({"detail": exc.detail}, exc.status_code)(scope, receive, send)
except PoolUnavailable as exc:
if creating and pool.local_threads.get(match[1], 0):
return await JSONResponse({"detail": str(exc)}, 409)(scope, receive, send)
return await JSONResponse({"detail": str(exc)}, 503, headers={"Retry-After": "3"})(scope, receive, send)
if node is None:
return await self.app(scope, receive, send)
if node.get("_fallback_to_main"):
fallback_worker = node.get("_worker")
if creating and fallback_worker is not None:
try:
active_response = await pool.http.get(
fallback_worker["url"] + "/internal/worker-thread-active/" + quote(match[1], safe=""),
headers={"x-deerflow-worker-token": pool.store.raw()["token"]},
timeout=2,
)
active_response.raise_for_status()
if active_response.json().get("active"):
return await JSONResponse({"detail": "该会话正在 Worker 执行,请等待当前回答完成"}, 409)(scope, receive, send)
except httpx.HTTPError:
return await JSONResponse({"detail": "无法确认该会话是否正在 Worker 执行,请稍后重试"}, 503, headers={"Retry-After": "2"})(scope, receive, send)
if not _main_has_persistent_chat_storage(scope["app"]):
return await JSONResponse(
{"detail": "主服务的持久化聊天存储当前不可用,无法安全接管 Worker 请求"},
503,
headers={"Retry-After": "3"},
)(scope, receive, send)
scope.setdefault("state", {})["worker_pool_fallback"] = True
scope["state"]["worker_pool_fallback_reason"] = node.get("_fallback_reason")
return await self.app(scope, receive, send)
response = None
scope.setdefault("state", {})["worker_node"] = node["name"]
try:
headers = [(k, v) for k, v in scope["headers"] if k.lower() not in HOP_HEADERS or k.lower() == b"host"]
headers.append((b"x-deerflow-worker-token", pool.store.raw()["token"].encode()))
url = node["url"] + scope["path"]
if scope.get("query_string"):
url += "?" + scope["query_string"].decode("ascii")
upstream_started = time.perf_counter()
response = await send_upstream(pool.http, scope["method"], url, headers, request.stream())
scope["state"]["worker_upstream_headers_ms"] = round((time.perf_counter() - upstream_started) * 1000, 2)
async def chunks():
async for chunk in response.aiter_raw():
yield chunk
downstream = StreamingResponse(chunks(), status_code=response.status_code)
downstream.raw_headers = [(k, v) for k, v in response.headers.raw if k.lower() not in HOP_HEADERS]
await downstream(scope, receive, send)
except httpx.HTTPError as exc:
logging.getLogger(__name__).warning("Worker transport failed method=%s node=%s error=%s", scope["method"], node["name"], type(exc).__name__)
if response is None:
await JSONResponse({"detail": "执行节点连接失败。任务可能已开始,请查看会话状态后再决定是否重试。"}, 503)(scope, receive, send)
# Once SSE headers are sent, close the stream so the SDK reconnects.
finally:
if response is not None:
await response.aclose()
pool.release(node["name"])
@@ -0,0 +1,52 @@
"""Observe shared skill edits before accepting a new model run."""
import asyncio
import time
class SharedSkillRefresh:
def __init__(self):
self.lock = asyncio.Lock()
self.extensions = None
self.skills = None
self.checked_at = 0.0
@staticmethod
def extension_stamp():
from deerflow.config.extensions_config import ExtensionsConfig
path = ExtensionsConfig.resolve_config_path()
if path is None:
return None
stat = path.stat()
return (str(path), stat.st_mtime_ns, stat.st_size)
@staticmethod
def skill_stamp():
from pathlib import Path
from deerflow.skills.storage import get_or_new_skill_storage
root = Path(get_or_new_skill_storage().get_skills_root_path())
stamps = []
for category in ("public", "custom"):
for path in (root / category).rglob("*"):
if path.suffix in {".md", ".py"} and path.is_file():
stat = path.stat()
stamps.append((str(path), stat.st_mtime_ns, stat.st_size))
return tuple(sorted(stamps))
async def check(self):
async with self.lock:
extension = await asyncio.to_thread(self.extension_stamp)
skills = self.skills
if time.monotonic() - self.checked_at >= 5:
skills = await asyncio.to_thread(self.skill_stamp)
self.checked_at = time.monotonic()
if extension != self.extensions or skills != self.skills:
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.extensions_config import reload_extensions_config
await asyncio.to_thread(reload_extensions_config)
await refresh_skills_system_prompt_cache_async()
self.extensions, self.skills = extension, skills
@@ -0,0 +1,12 @@
"""Fingerprint this worker's execution adapter at process startup."""
import hashlib
from pathlib import Path
def code_revision():
digest = hashlib.sha256()
for path in sorted(Path(__file__).parent.glob("*.py")):
digest.update(path.name.encode())
digest.update(path.read_bytes())
return digest.hexdigest()[:12]
@@ -0,0 +1,133 @@
"""Strict admin-only management; anonymous mode never grants host access."""
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel
from app.gateway.deps import get_current_user_from_request
from .controller import PoolUnavailable
from .docker import DockerError
from .preflight import shared_storage_configured
from .settings import PoolSettings
router = APIRouter(prefix="/api/system-settings/worker-pool", tags=["worker-pool"])
class DiagnosticsSwitch(BaseModel):
enabled: bool
@router.get("/diagnostics")
async def diagnostics_status(request: Request):
return (await controller(request)).diagnostics.status()
@router.put("/diagnostics")
async def diagnostics_switch(body: DiagnosticsSwitch, request: Request):
pool = await controller(request)
await pool.diagnostics.set_enabled(body.enabled)
return pool.diagnostics.status()
@router.get("/logs")
async def container_logs(request: Request, node: str = Query(min_length=1, max_length=128), tail: int = Query(default=200, ge=1, le=1000), minutes: int = Query(default=60, ge=1, le=1440), query: str = Query(default="", max_length=100)):
pool = await controller(request)
try:
return await pool.diagnostics.logs(node, tail, minutes, query)
except DockerError as exc:
raise HTTPException(400, str(exc)) from None
@router.get("/log-archives")
async def log_archives(request: Request):
return (await controller(request)).diagnostics.archives()
async def controller(request: Request):
user = await get_current_user_from_request(request)
if user is None or getattr(user, "system_role", None) != "admin":
raise HTTPException(403, "仅管理员可以管理执行节点")
from .lifecycle import ensure_controller
return ensure_controller(request.app)
@router.get("")
async def status(request: Request):
return (await controller(request)).status()
@router.put("")
async def save(body: PoolSettings, request: Request):
pool = await controller(request)
async with pool.lock:
current = pool.store.load()
immutable = ("transport", "host", "port", "username", "source_container", "network", "concurrency", "cpu", "memory_mb", "queue_limit", "queue_timeout", "python_executable", "working_dir", "database_pool_size", "postgres_pool_size")
controller_active = pool.task is not None and not pool.task.done()
if (pool.nodes or (controller_active and not pool.discovery_complete)) and any(getattr(current, key) != getattr(body, key) for key in immutable):
raise HTTPException(409, "请先将节点数设为 0,等待所有节点退出后再修改连接或单节点配置")
config = getattr(request.app.state, "config", None)
if body.enabled:
if not shared_storage_configured(config):
raise HTTPException(409, "启用前必须配置持久化 PostgreSQL 或 SQLite 聊天存储及共享业务库")
from langgraph.checkpoint.memory import InMemorySaver
if isinstance(getattr(request.app.state, "checkpointer", None), InMemorySaver):
raise HTTPException(409, "当前检查点已降级到内存,不能启用 Worker")
if "AioSandboxProvider" in str(getattr(getattr(config, "sandbox", None), "use", "")):
raise HTTPException(409, "当前沙箱需要 Docker 管理权限,在线 Worker 暂不支持;请先配置不依赖 Docker socket 的沙箱")
if body.desired * body.cpu > body.total_cpu or body.desired * body.memory_mb > body.total_memory_mb:
raise HTTPException(422, "Worker 总资源超过预算")
if body.desired * (body.database_pool_size + 2 * body.postgres_pool_size) > body.database_connection_budget:
raise HTTPException(422, "Worker 数据库连接总额度超过预算")
identity = ("transport", "host", "port", "username", "host_key_sha256")
if body.transport == "ssh" and body.password is None and current.password and any(getattr(body, k) != getattr(current, k) for k in identity):
raise HTTPException(422, "服务器连接身份已改变,请重新填写密码")
pool.store.save(body)
try:
pool.start()
except RuntimeError as exc:
pool.store.save(current)
raise HTTPException(409, str(exc)) from None
return pool.status()
@router.post("/test")
async def test_connection(request: Request):
pool = await controller(request)
try:
async with pool.lock:
await pool.inspect_template(pool.store.load())
except (DockerError, PoolUnavailable) as exc:
raise HTTPException(400, str(exc)) from None
return {"ok": True, "message": "服务器连接和容器模板检查通过,镜像内 Python 路径已自动校验;节点启动后还会检查聊天存储与运行状态"}
@router.post("/fingerprint")
async def fingerprint(body: PoolSettings, request: Request):
import asyncio
from .discovery import host_fingerprint
await controller(request)
try:
value = await asyncio.to_thread(host_fingerprint, body.host, body.port)
except DockerError as exc:
raise HTTPException(400, str(exc)) from None
return {"fingerprint": value}
@router.post("/discover")
async def discover_server(body: PoolSettings, request: Request):
import asyncio
from .discovery import discover
pool = await controller(request)
current = pool.store.load()
if body.password is None:
identity = ("transport", "host", "port", "username", "host_key_sha256")
if all(getattr(body, k) == getattr(current, k) for k in identity):
body = body.model_copy(update={"password": current.password})
try:
return await asyncio.to_thread(discover, body, pool.store.raw()["instance"])
except DockerError as exc:
raise HTTPException(400, str(exc)) from None
@@ -0,0 +1,54 @@
"""Runtime checks shared by the controller's interpreter probe and workers."""
from __future__ import annotations
import importlib
def storage_backend(config) -> str:
"""Return the effective LangGraph persistence adapter selected by config."""
checkpoint = getattr(config, "checkpointer", None)
if checkpoint is not None:
if checkpoint.type == "postgres":
from deerflow.runtime.checkpointer.async_provider import _use_postgres_enabled
return "postgres" if _use_postgres_enabled(checkpoint) else "sqlite"
if checkpoint.type == "sqlite":
return "sqlite"
return "memory"
database = getattr(config, "database", None)
backend = getattr(database, "backend", "memory")
return backend if backend in {"postgres", "sqlite"} else "memory"
def verify_storage_dependencies(config) -> str:
"""Verify the adapters the configured runtime will import, without DB access."""
backend = storage_backend(config)
modules = {
"postgres": (
"langgraph.checkpoint.postgres.aio",
"langgraph.store.postgres.aio",
"psycopg.rows",
"psycopg_pool",
),
"sqlite": (
"langgraph.checkpoint.sqlite.aio",
"langgraph.store.sqlite.aio",
),
}.get(backend, ())
for module in modules:
try:
importlib.import_module(module)
except ImportError as exc:
install = (
"langgraph-checkpoint-postgres psycopg[binary] psycopg-pool"
if backend == "postgres"
else "langgraph-checkpoint-sqlite aiosqlite"
)
raise RuntimeError(
f"Worker Python is missing {module} required by the configured "
f"{backend} chat store; install {install} in the main service "
"runtime and ensure Workers use that same Python environment"
) from exc
return backend
@@ -0,0 +1,95 @@
"""Private controller configuration. Never expose credentials through GET APIs."""
from __future__ import annotations
import json
import os
import re
import secrets
from pathlib import Path
from typing import Literal
from cryptography.fernet import Fernet
from pydantic import BaseModel, Field, field_validator
class PoolSettings(BaseModel):
enabled: bool = False
transport: Literal["ssh", "socket"] = "ssh"
host: str = ""
port: int = Field(default=22, ge=1, le=65535)
username: str = ""
host_key_sha256: str = ""
source_container: str = ""
network: str = ""
python_executable: str = Field(default="/opt/venv/bin/python", pattern=r"^/[a-zA-Z0-9_./-]+$")
working_dir: str = Field(default="/app/backend", pattern=r"^/[a-zA-Z0-9_./-]+$")
desired: int = Field(default=0, ge=0, le=32)
concurrency: int = Field(default=4, ge=1, le=32)
database_pool_size: int = Field(default=4, ge=1, le=32)
postgres_pool_size: int = Field(default=4, ge=1, le=32)
database_connection_budget: int = Field(default=64, ge=3, le=1024)
queue_limit: int = Field(default=128, ge=1, le=1024)
queue_timeout: int = Field(default=60, ge=1, le=300)
cpu: float = Field(default=2, ge=0.25, le=64)
memory_mb: int = Field(default=2048, ge=512, le=131072)
total_cpu: float = Field(default=8, ge=0.25, le=512)
total_memory_mb: int = Field(default=8192, ge=512, le=1048576)
password: str | None = Field(default=None, max_length=4096, repr=False)
@field_validator("source_container", "network")
@classmethod
def docker_name(cls, value: str) -> str:
if value and not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}", value):
raise ValueError("容器名和网络名只能包含字母、数字、点、横线、下划线")
return value
class SettingsStore:
def __init__(self, directory: Path | None = None):
self.directory = directory or Path(os.getenv("DEER_FLOW_WORKER_CONTROL_HOME", ".worker-pool")).resolve()
self.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
key_path = self.directory / "secret.key"
try:
fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
pass
else:
with os.fdopen(fd, "wb") as out:
out.write(Fernet.generate_key())
self.cipher = Fernet(key_path.read_bytes())
self.path = self.directory / "settings.json"
if not self.path.exists():
self._write({"instance": secrets.token_hex(8), "token": secrets.token_urlsafe(48), "settings": {}})
def _write(self, data: dict) -> None:
temp = self.path.with_suffix(".tmp")
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as out:
json.dump(data, out)
out.flush()
os.fsync(out.fileno())
os.replace(temp, self.path)
def raw(self) -> dict:
return json.loads(self.path.read_text(encoding="utf-8"))
def load(self) -> PoolSettings:
data = self.raw()["settings"].copy()
encrypted = data.pop("encrypted_password", None)
if encrypted:
data["password"] = self.cipher.decrypt(encrypted.encode()).decode()
return PoolSettings(**data)
def save(self, settings: PoolSettings) -> None:
data = self.raw()
values = settings.model_dump(exclude={"password"})
password = settings.password if settings.password is not None else self.load().password
if password:
values["encrypted_password"] = self.cipher.encrypt(password.encode()).decode()
data["settings"] = values
self._write(data)
def public(self) -> dict:
settings = self.load()
return {**settings.model_dump(exclude={"password"}), "password_set": bool(settings.password)}
@@ -0,0 +1,258 @@
"""Opt-in, bounded HTTP timing. Never consumes/rebuffers a streaming body."""
import logging
import math
import re
import time
import uuid
from collections import deque
from app.gateway.perf_logging import is_verbose_path, should_log_request, verbose_request_logging
logger = logging.getLogger(__name__)
def route_name(path):
path = re.sub(r"(/api/threads/)(?!search(?:/|$)|count(?:/|$))[^/]+", r"\1{thread_id}", path)
path = re.sub(r"(/runs/)[^/]+(?=/|$)", r"\1{run_id}", path) if not path.endswith(("/runs/stream", "/runs/wait")) else path
return re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27,}", "{id}", path, flags=re.I)[:240]
def percentile(values, percent):
values = sorted(v for v in values if v is not None)
return round(values[max(0, math.ceil(len(values) * percent / 100) - 1)], 2) if values else None
def _database_pool_snapshot():
try:
from deerflow.persistence.engine import get_engine
engine = get_engine()
if engine is None:
return None
pool = engine.sync_engine.pool
return {
"backend": engine.dialect.name,
"checked_out": pool.checkedout(),
"size": pool.size(),
"overflow": pool.overflow(),
"max_overflow": getattr(pool, "_max_overflow", None),
}
except Exception:
return None
class RequestRecorder:
def __init__(self):
self.enabled = False
self.completed = deque(maxlen=2000)
self.pending = deque(maxlen=4000)
self.active = {}
def finish(self, record):
self.completed.append(record)
self.pending.append(record)
self.active.pop(record["id"], None)
def summary(self):
now = time.time()
rows = [r for r in self.completed if r["started"] >= now - 900]
grouped = {}
for r in rows:
# Preserve where the request actually ran. A gateway-only
# aggregate hides a saturated Worker behind a healthy main node.
key = (r.get("node") or "main", r["method"], r["path"], r["stream"])
grouped.setdefault(key, []).append(r)
aggregates = []
for (node, method, path, stream), records in grouped.items():
aggregates.append({"node": node, "method": method, "path": path, "stream": stream, "count": len(records),
"errors": sum(r["status"] >= 400 for r in records),
"p95_headers_ms": percentile([r["headers_ms"] for r in records], 95),
"p95_ms": percentile([r["total_ms"] for r in records], 95),
"max_ms": max(r["total_ms"] for r in records)})
return {"window_seconds": 900, "count": len(rows), "errors": sum(r["status"] >= 400 for r in rows),
"p95_headers_ms": percentile([r["headers_ms"] for r in rows], 95),
"routes": sorted(aggregates, key=lambda r: r["max_ms"], reverse=True)[:40],
"slow_requests": sorted(rows, key=lambda r: r["total_ms"], reverse=True)[:30],
"active": [{**r, "elapsed_ms": round((now - r["started"]) * 1000, 1)} for r in list(self.active.values())[:100]]}
def worker_status(self):
"""Bounded status payload for the main process' 5-second health probe."""
summary = self.summary()
return {
"count": summary["count"],
"errors": summary["errors"],
"p95_headers_ms": summary["p95_headers_ms"],
"routes": summary["routes"][:8],
"active_count": len(self.active),
}
class RequestTelemetryMiddleware:
def __init__(self, app):
self.app = app
self._active_requests = 0
self._active_by_path = {}
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
recorder = getattr(scope["app"].state, "request_recorder", None)
path = scope.get("path", "")
if not path.startswith("/api/") or path.startswith(("/api/system-settings/worker-pool", "/api/health")):
return await self.app(scope, receive, send)
record_request = recorder is not None and recorder.enabled and len(recorder.active) < 2000
begin = time.perf_counter()
request_id = uuid.uuid4().hex[:12]
route_key = route_name(path)
self._active_requests += 1
self._active_by_path[route_key] = self._active_by_path.get(route_key, 0) + 1
api_inflight_start = self._active_requests
route_inflight_start = self._active_by_path[route_key]
record = {"id": request_id, "started": time.time(), "method": scope["method"], "path": route_name(path),
"status": 499, "headers_ms": None, "first_body_ms": None, "stream": False} if record_request else None
if record is not None:
recorder.active[record["id"]] = record
state = scope.setdefault("state", {})
state["diagnostic_request_id"] = request_id
state["perf_api_inflight_start"] = api_inflight_start
state["perf_route_inflight_start"] = route_inflight_start
request_context_token = None
try:
from deerflow.persistence.engine import bind_performance_request
request_context_token = bind_performance_request(request_id)
except Exception:
logger.debug("PERF request ID could not be attached to the SQL context", exc_info=True)
async def measured_send(message):
if message["type"] == "http.response.start":
headers_ms = round((time.perf_counter() - begin) * 1000, 2)
headers = list(message.get("headers", []))
is_stream = any(k.lower() == b"content-type" and b"text/event-stream" in v for k, v in headers)
state["perf_headers_ms"] = headers_ms
state["perf_stream"] = is_stream
state["perf_status"] = message["status"]
if record is not None:
record["status"] = message["status"]
record["headers_ms"] = headers_ms
record["stream"] = is_stream
# Record existing operation phases without query text or headers.
phases = next((v.decode("latin-1")[:1000] for k, v in headers if k.lower() == b"server-timing"), "")
if record is not None:
record["phases"] = re.sub(r"[^a-zA-Z0-9_.,;= -]", "", phases)
headers.append((b"x-deerflow-request-id", request_id.encode()))
message = {**message, "headers": headers}
elif message["type"] == "http.response.body" and message.get("body") and state.get("perf_first_body_ms") is None:
state["perf_first_body_ms"] = round((time.perf_counter() - begin) * 1000, 2)
if record is not None:
record["first_body_ms"] = state["perf_first_body_ms"]
await send(message)
try:
await self.app(scope, receive, measured_send)
except BaseException as exc:
cancelled = type(exc).__name__.lower() in {"cancelled", "cancellederror"} or "cancelled" in type(exc).__name__.lower()
if record is not None and record["headers_ms"] is None:
record["status"] = 499 if cancelled else 500
state["perf_exception_type"] = type(exc).__name__
state["perf_cancelled"] = cancelled
if record is None or record["headers_ms"] is None:
state["perf_status"] = 499 if cancelled else 500
raise
finally:
total_ms = round((time.perf_counter() - begin) * 1000, 2)
route = scope.get("route")
route_path = getattr(route, "path", None) or route_name(path)
route_path = route_path[:240]
headers_ms = record.get("headers_ms") if record is not None else state.get("perf_headers_ms")
is_stream = bool(state.get("perf_stream", record.get("stream") if record is not None else False))
status = record.get("status", 500) if record is not None else state.get("perf_status", 500)
exception_type = state.get("perf_exception_type")
node = state.get("worker_node", "main")
trace = state.get("perf_trace")
threshold = should_log_request(
elapsed_ms=total_ms,
headers_ms=headers_ms,
first_body_ms=state.get("perf_first_body_ms"),
is_stream=is_stream,
)
verbose = verbose_request_logging() and is_verbose_path(path)
cancelled = state.get("perf_cancelled", False)
concurrency_detail = api_inflight_start >= 8
detailed = threshold or verbose or status >= 500 or (exception_type is not None and not cancelled) or concurrency_detail
if detailed:
endpoint_ms = trace.get("endpoint_ms") if isinstance(trace, dict) else None
pool_snapshot = _database_pool_snapshot()
level = logging.ERROR if not cancelled and (status >= 500 or exception_type is not None) else logging.WARNING if threshold else logging.INFO
logger.log(
level,
"PERF http_request id=%s method=%s path=%s status=%s total_ms=%.2f headers_ms=%s first_body_ms=%s stream=%s "
"api_inflight_start=%s route_inflight_start=%s api_inflight_end=%s route_inflight_end=%s db_pool=%s node=%s "
"worker_access_ms=%s worker_upstream_headers_ms=%s worker_fallback=%s event_loop_lag_ms=%s "
"exception_type=%s endpoint=%s endpoint_ms=%s outside_handler_ms=%s stages=%s",
request_id,
scope["method"],
route_path,
status,
total_ms,
headers_ms,
state.get("perf_first_body_ms"),
is_stream,
api_inflight_start,
route_inflight_start,
self._active_requests,
self._active_by_path.get(route_key, 0),
pool_snapshot,
node,
state.get("worker_access_ms"),
state.get("worker_upstream_headers_ms"),
state.get("worker_pool_fallback_reason"),
getattr(scope["app"].state, "perf_event_loop_lag_ms", None),
exception_type,
trace.get("endpoint") if isinstance(trace, dict) else None,
endpoint_ms,
round(max(0.0, headers_ms - endpoint_ms), 2) if headers_ms is not None and endpoint_ms is not None else None,
{"stages": trace.get("stages"), "details": trace.get("details")} if isinstance(trace, dict) else None,
)
else:
logger.info(
"PERF http_request id=%s method=%s path=%s status=%s total_ms=%.2f headers_ms=%s api_inflight_start=%s route_inflight_start=%s api_inflight_end=%s route_inflight_end=%s node=%s fallback=%s exception_type=%s",
request_id,
scope["method"],
route_path,
status,
total_ms,
headers_ms,
api_inflight_start,
route_inflight_start,
self._active_requests,
self._active_by_path.get(route_key, 0),
node,
state.get("worker_pool_fallback_reason"),
exception_type,
)
state["perf_status"] = status
if record is not None:
record["total_ms"] = total_ms
record["node"] = node
record["access_ms"] = state.get("worker_access_ms")
record["upstream_headers_ms"] = state.get("worker_upstream_headers_ms")
record["fallback_reason"] = state.get("worker_pool_fallback_reason")
if record is not None:
record["path"] = route_path
recorder.finish(record)
if request_context_token is not None:
try:
from deerflow.persistence.engine import reset_performance_request
reset_performance_request(request_context_token)
except Exception:
logger.debug("PERF request ID could not be cleared from the SQL context", exc_info=True)
self._active_requests = max(0, self._active_requests - 1)
remaining_for_path = self._active_by_path.get(route_key, 0) - 1
if remaining_for_path > 0:
self._active_by_path[route_key] = remaining_for_path
else:
self._active_by_path.pop(route_key, None)
@@ -0,0 +1,135 @@
"""Dedicated single-process online execution entrypoint."""
from __future__ import annotations
import asyncio
import hmac
import os
import time
from contextlib import asynccontextmanager
from starlette.responses import JSONResponse
from app.gateway.app import create_app
from app.gateway.deps import langgraph_runtime
from app.gateway.runtime_executor import configure_runtime_executor
from deerflow.config import get_app_config
from .admission import Admission
from .manager import WorkerRunManager
from .preflight import shared_storage_configured, validate_sqlite_mount
from .proxy import THREAD_PATH
from .refresh import SharedSkillRefresh
async def _measure_worker_event_loop_lag(app):
loop = asyncio.get_running_loop()
while True:
if not getattr(app.state, "worker_diagnostics_enabled", False):
app.state.worker_event_loop_lag_ms = 0.0
await asyncio.sleep(1)
continue
started = loop.time()
await asyncio.sleep(1)
app.state.worker_event_loop_lag_ms = round(max(0.0, loop.time() - started - 1) * 1000, 2)
@asynccontextmanager
async def worker_lifespan(app):
from .revision import code_revision
app.state.worker_process_started = time.time()
app.state.worker_runtime_revision = code_revision()
config = get_app_config()
from .runtime import verify_storage_dependencies
verify_storage_dependencies(config)
if not shared_storage_configured(config):
raise RuntimeError("Managed workers require persistent PostgreSQL or SQLite chat storage")
if "AioSandboxProvider" in str(getattr(config.sandbox, "use", "")):
raise RuntimeError("Managed online workers cannot launch Docker sandboxes")
validate_sqlite_mount(config)
# Shared event records follow conversations across drained containers.
config = config.model_copy(
update={
"run_events": config.run_events.model_copy(update={"backend": "db"}),
"database": config.database.model_copy(update={"pool_size": int(os.environ.get("DEER_FLOW_WORKER_DB_POOL_SIZE", "4")), "max_overflow": 0}),
}
)
app.state.config = config
app.state.worker_skill_refresh = SharedSkillRefresh()
app.state.worker_deferred_admission = True
app.state.worker_admission = Admission(int(os.environ.get("DEER_FLOW_WORKER_CONCURRENCY", "4")), int(os.environ.get("DEER_FLOW_WORKER_QUEUE_LIMIT", "128")), float(os.environ.get("DEER_FLOW_WORKER_QUEUE_TIMEOUT", "60")))
# Each execution container keeps its own bounded rolling request metrics;
# the controller reads a compact summary with the normal status probe.
from .telemetry import RequestRecorder
app.state.request_recorder = RequestRecorder()
app.state.worker_diagnostics_enabled = False
app.state.worker_event_loop_lag_ms = 0.0
await configure_runtime_executor(config.runtime_performance)
async with langgraph_runtime(app, background=False):
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.store.memory import InMemoryStore
if isinstance(app.state.checkpointer, InMemorySaver) or isinstance(app.state.store, InMemoryStore):
raise RuntimeError("Managed worker refuses an in-memory persistence fallback")
close_manager = getattr(app.state.run_manager, "aclose", None)
if close_manager is not None:
await close_manager()
app.state.run_manager = WorkerRunManager(store=app.state.run_store)
retention = getattr(config, "runtime_contention", None)
if retention is not None and hasattr(app.state.run_manager, "start_retention"):
app.state.run_manager.start_retention(retention_seconds=retention.completed_run_retention_seconds, interval_seconds=retention.run_sweep_seconds)
lag_task = asyncio.create_task(_measure_worker_event_loop_lag(app))
try:
yield
finally:
lag_task.cancel()
await asyncio.gather(lag_task, return_exceptions=True)
class InternalOnly:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
expected = os.environ.get("DEER_FLOW_WORKER_TOKEN", "")
token = dict(scope["headers"]).get(b"x-deerflow-worker-token", b"").decode()
if not expected or not hmac.compare_digest(token, expected):
return await JSONResponse({"detail": "Forbidden"}, 403)(scope, receive, send)
if scope["path"] == "/internal/worker-status":
gate = scope["app"].state.worker_admission
recorder = getattr(scope["app"].state, "request_recorder", None)
diagnostics_enabled = dict(scope["headers"]).get(b"x-deerflow-diagnostics-enabled") == b"true"
scope["app"].state.worker_diagnostics_enabled = diagnostics_enabled
if recorder is not None:
recorder.enabled = diagnostics_enabled
http = recorder.worker_status() if recorder is not None else None
return await JSONResponse({"ready": True, "active": gate.active + gate.waiting, "running": gate.active, "waiting": gate.waiting, "capacity": gate.capacity,
"process_started_at": scope["app"].state.worker_process_started,
"runtime_revision": scope["app"].state.worker_runtime_revision,
"event_loop_lag_ms": getattr(scope["app"].state, "worker_event_loop_lag_ms", None),
"diagnostics_enabled": diagnostics_enabled,
"http": http})(scope, receive, send)
active_prefix = "/internal/worker-thread-active/"
if scope["path"].startswith(active_prefix):
from urllib.parse import unquote
thread_id = unquote(scope["path"][len(active_prefix):])
active = scope["app"].state.run_manager.has_active_thread(thread_id)
return await JSONResponse({"active": active})(scope, receive, send)
match = THREAD_PATH.fullmatch(scope["path"])
if not match:
return await JSONResponse({"detail": "Worker endpoint not available"}, 404)(scope, receive, send)
scope.setdefault("state", {})["worker_node"] = os.environ.get("HOSTNAME", "worker")
from deerflow.runtime.thread_paths import aresolve_path_user_id
await aresolve_path_user_id(match[1])
await self.app(scope, receive, send)
app = create_app()
app.router.lifespan_context = worker_lifespan
app.add_middleware(InternalOnly)