sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Single-host managed online workers; background scheduling stays on the gateway."""
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Bound executing runs, including detached/background runs, not HTTP streams."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
_worker_ticket = ContextVar("worker_admission_ticket", default=None)
|
||||
|
||||
|
||||
class AdmissionTicket:
|
||||
def __init__(self, gate):
|
||||
if gate.active + gate.waiting >= gate.capacity + gate.queue_limit:
|
||||
raise HTTPException(429, "执行节点繁忙,请稍后重试", headers={"Retry-After": "3"})
|
||||
self.gate = gate
|
||||
self.deadline = asyncio.get_running_loop().time() + gate.timeout
|
||||
self.waiting = True
|
||||
self.acquired = False
|
||||
self.released = False
|
||||
self.started = False
|
||||
gate.waiting += 1
|
||||
|
||||
async def acquire(self):
|
||||
try:
|
||||
remaining = max(0.001, self.deadline - asyncio.get_running_loop().time())
|
||||
await asyncio.wait_for(self.gate.semaphore.acquire(), remaining)
|
||||
self.acquired = True
|
||||
self.gate.active += 1
|
||||
except TimeoutError:
|
||||
raise HTTPException(429, "等待执行超时,请稍后重试") from None
|
||||
finally:
|
||||
if self.waiting:
|
||||
self.gate.waiting -= 1
|
||||
self.waiting = False
|
||||
|
||||
def release(self, _task=None):
|
||||
if self.released:
|
||||
return
|
||||
self.released = True
|
||||
if self.waiting:
|
||||
self.gate.waiting -= 1
|
||||
self.waiting = False
|
||||
if self.acquired:
|
||||
self.gate.active -= 1
|
||||
self.gate.semaphore.release()
|
||||
|
||||
|
||||
async def _finish_queued_run(kwargs, *, cancelled=False, message="等待执行超时,请稍后重试"):
|
||||
from deerflow.runtime.runs.schemas import RunStatus
|
||||
|
||||
manager, record, bridge = kwargs["run_manager"], kwargs["record"], kwargs["bridge"]
|
||||
try:
|
||||
async with asyncio.timeout(5):
|
||||
await manager.set_status(record.run_id, RunStatus.interrupted if cancelled else RunStatus.error, error=None if cancelled else message)
|
||||
ctx = kwargs.get("ctx")
|
||||
store = getattr(ctx, "thread_store", None)
|
||||
if store is not None and not getattr(record, "superseded", False):
|
||||
await store.update_status(record.thread_id, "interrupted" if cancelled else "error")
|
||||
finally:
|
||||
try:
|
||||
if not cancelled:
|
||||
await bridge.publish(record.run_id, "error", {"name": "WorkerQueueTimeout", "message": message})
|
||||
finally:
|
||||
await bridge.publish_end(record.run_id)
|
||||
asyncio.create_task(bridge.cleanup(record.run_id, delay=60))
|
||||
|
||||
|
||||
async def run_with_worker_admission(runner, **kwargs):
|
||||
ticket = _worker_ticket.get()
|
||||
if ticket is None:
|
||||
return await runner(**kwargs)
|
||||
ticket.started = True
|
||||
record, bridge = kwargs["record"], kwargs["bridge"]
|
||||
# Publish a real, persisted run id before waiting for an execution slot.
|
||||
# The router keeps its canonical Location/reconnect headers unchanged.
|
||||
try:
|
||||
await bridge.publish(record.run_id, "metadata", {"run_id": record.run_id, "thread_id": record.thread_id,
|
||||
"worker_queue": {"waiting": ticket.gate.waiting, "capacity": ticket.gate.capacity}})
|
||||
await ticket.acquire()
|
||||
except asyncio.CancelledError:
|
||||
await _finish_queued_run(kwargs, cancelled=True)
|
||||
raise
|
||||
except HTTPException as exc:
|
||||
await _finish_queued_run(kwargs, message=str(exc.detail))
|
||||
return
|
||||
except Exception:
|
||||
await _finish_queued_run(kwargs, message="执行节点排队初始化失败,请查看节点日志")
|
||||
return
|
||||
return await runner(**kwargs)
|
||||
|
||||
|
||||
class Admission:
|
||||
def __init__(self, capacity: int, queue_limit: int = 128, timeout: float = 60):
|
||||
self.capacity = capacity
|
||||
self.active = 0
|
||||
self.waiting = 0
|
||||
self.queue_limit = queue_limit
|
||||
self.timeout = timeout
|
||||
self.semaphore = asyncio.Semaphore(capacity)
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self):
|
||||
async with self.lock:
|
||||
if self.waiting >= self.queue_limit:
|
||||
raise HTTPException(429, "执行节点繁忙,请稍后重试", headers={"Retry-After": "3"})
|
||||
self.waiting += 1
|
||||
try:
|
||||
await asyncio.wait_for(self.semaphore.acquire(), self.timeout)
|
||||
except TimeoutError:
|
||||
raise HTTPException(429, "等待执行超时,请稍后重试", headers={"Retry-After": "3"}) from None
|
||||
finally:
|
||||
self.waiting -= 1
|
||||
self.active += 1
|
||||
|
||||
def release(self, _task=None):
|
||||
self.active -= 1
|
||||
self.semaphore.release()
|
||||
|
||||
|
||||
def bounded_worker_run(function):
|
||||
@wraps(function)
|
||||
async def wrapped(body, thread_id, request, *args, **kwargs):
|
||||
pool = getattr(request.app.state, "worker_pool", None)
|
||||
fallback_to_main = bool(getattr(getattr(request, "state", None), "worker_pool_fallback", False))
|
||||
if pool is not None and pool.db is not None and pool.db.execute("SELECT 1 FROM affinity WHERE thread=?", (thread_id,)).fetchone() and not fallback_to_main:
|
||||
raise HTTPException(409, "该会话由独立 Worker 管理,请通过会话接口提交,不能在主服务重复执行")
|
||||
gate = getattr(request.app.state, "worker_admission", None)
|
||||
if gate is None:
|
||||
if pool is None:
|
||||
return await function(body, thread_id, request, *args, **kwargs)
|
||||
if pool.local_threads.get(thread_id, 0):
|
||||
raise HTTPException(409, "该会话正在主服务执行,请等待当前回答完成")
|
||||
# Reserve before the first await, including start_run's preparation
|
||||
# phase before RunRecord.task exists. Otherwise a concurrent routed
|
||||
# request could assign this same thread to another process.
|
||||
pool.local_threads[thread_id] = pool.local_threads.get(thread_id, 0) + 1
|
||||
|
||||
def release_local(_task=None):
|
||||
remaining = pool.local_threads.get(thread_id, 1) - 1
|
||||
if remaining:
|
||||
pool.local_threads[thread_id] = remaining
|
||||
else:
|
||||
pool.local_threads.pop(thread_id, None)
|
||||
|
||||
try:
|
||||
record = await function(body, thread_id, request, *args, **kwargs)
|
||||
except BaseException:
|
||||
release_local()
|
||||
raise
|
||||
if record.task is None:
|
||||
release_local()
|
||||
else:
|
||||
record.task.add_done_callback(release_local)
|
||||
return record
|
||||
if getattr(request.app.state, "worker_deferred_admission", False) and getattr(request.app.state, "concurrency_gate", None) is None:
|
||||
ticket = AdmissionTicket(gate)
|
||||
token = _worker_ticket.set(ticket)
|
||||
try:
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
|
||||
await aresolve_path_user_id(thread_id)
|
||||
refresh = getattr(request.app.state, "worker_skill_refresh", None)
|
||||
if refresh is not None:
|
||||
await refresh.check()
|
||||
record = await function(body, thread_id, request, *args, **kwargs)
|
||||
except BaseException:
|
||||
ticket.release()
|
||||
raise
|
||||
finally:
|
||||
_worker_ticket.reset(token)
|
||||
if record.task is None:
|
||||
ticket.release()
|
||||
else:
|
||||
def finish_ticket(task):
|
||||
ticket.release()
|
||||
if task.cancelled() and not ticket.started:
|
||||
manager = getattr(request.app.state, "run_manager", None)
|
||||
bridge = getattr(request.app.state, "stream_bridge", None)
|
||||
if manager is not None and bridge is not None:
|
||||
asyncio.create_task(_finish_queued_run({"run_manager": manager, "record": record, "bridge": bridge}, cancelled=True))
|
||||
record.task.add_done_callback(finish_ticket)
|
||||
return record
|
||||
await gate.acquire()
|
||||
try:
|
||||
# File/path middleware is synchronous. Resolve the persisted owner
|
||||
# on the database's home loop before it touches those helpers.
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
|
||||
await aresolve_path_user_id(thread_id)
|
||||
refresh = getattr(request.app.state, "worker_skill_refresh", None)
|
||||
if refresh is not None:
|
||||
await refresh.check()
|
||||
record = await function(body, thread_id, request, *args, **kwargs)
|
||||
except BaseException:
|
||||
gate.release()
|
||||
raise
|
||||
if record.task is None:
|
||||
gate.release()
|
||||
else:
|
||||
record.task.add_done_callback(gate.release)
|
||||
return record
|
||||
|
||||
return wrapped
|
||||
@@ -0,0 +1,359 @@
|
||||
"""One gateway owns reconciliation and durable conversation affinity.
|
||||
|
||||
Never move conversations from a live worker. A draining worker is stopped only
|
||||
after its run registry AND outstanding HTTP requests are empty. Failure does
|
||||
not replay a request, because a model/tool may already have produced effects.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
from .docker import DockerEngine, DockerError, worker_spec
|
||||
from .settings import SettingsStore
|
||||
|
||||
|
||||
class PoolUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Controller:
|
||||
def __init__(self, store: SettingsStore):
|
||||
self.store = store
|
||||
self.lock = asyncio.Lock()
|
||||
self.route_lock = asyncio.Lock()
|
||||
self.nodes: dict[str, dict] = {}
|
||||
self.error: str | None = None
|
||||
self.task: asyncio.Task | None = None
|
||||
self._lock_handle = None
|
||||
self.http = httpx.AsyncClient(timeout=httpx.Timeout(15, read=None), trust_env=False, limits=httpx.Limits(max_connections=256, max_keepalive_connections=64))
|
||||
self.db: sqlite3.Connection | None = None
|
||||
self.pending = 0
|
||||
self.inflight: dict[str, int] = {}
|
||||
self.local_threads: dict[str, int] = {}
|
||||
self._python_probe_key: tuple | None = None
|
||||
self._python_probe_result: str | None = None
|
||||
self._worker_image_key: tuple | None = None
|
||||
self._worker_image_result: str | None = None
|
||||
self.discovery_complete = not bool(store.load().source_container)
|
||||
|
||||
def _ensure_db(self):
|
||||
if self.db is None:
|
||||
self.db = sqlite3.connect(self.store.directory / "affinity.sqlite3")
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS affinity (thread TEXT PRIMARY KEY, node TEXT NOT NULL)")
|
||||
self.db.commit()
|
||||
return self.db
|
||||
|
||||
def _acquire_controller_lock(self):
|
||||
if self._lock_handle is not None:
|
||||
return
|
||||
self.store.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
handle = (self.store.directory / "controller.lock").open("a+b")
|
||||
try:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
handle.seek(0)
|
||||
if not handle.read(1):
|
||||
handle.write(b"0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
raise RuntimeError("Worker 池只能由一个主服务进程管理;请关闭重复的 Worker 池控制器") from None
|
||||
self._lock_handle = handle
|
||||
|
||||
def start(self):
|
||||
settings = self.store.load()
|
||||
if settings.enabled:
|
||||
self._acquire_controller_lock()
|
||||
self._ensure_db()
|
||||
if self.task is None or self.task.done():
|
||||
self.task = asyncio.create_task(self.loop())
|
||||
if hasattr(self, "diagnostics") and self.diagnostics.enabled:
|
||||
self.diagnostics.start()
|
||||
|
||||
async def close(self):
|
||||
if self.task:
|
||||
self.task.cancel()
|
||||
await asyncio.gather(self.task, return_exceptions=True)
|
||||
if hasattr(self, "diagnostics"):
|
||||
await self.diagnostics.close()
|
||||
await self.http.aclose()
|
||||
if self.db is not None:
|
||||
self.db.close()
|
||||
self.db = None
|
||||
self._release_controller_lock()
|
||||
|
||||
async def loop(self):
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
await self.reconcile()
|
||||
except Exception as exc:
|
||||
self.error = str(exc) if isinstance(exc, (DockerError, PoolUnavailable)) else f"节点检查失败({type(exc).__name__})"
|
||||
settings = self.store.load()
|
||||
if not settings.enabled and not self.nodes:
|
||||
return
|
||||
await asyncio.sleep(5)
|
||||
finally:
|
||||
if not self.store.load().enabled:
|
||||
self._release_controller_lock()
|
||||
|
||||
def _release_controller_lock(self):
|
||||
if self._lock_handle is None:
|
||||
return
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
self._lock_handle.seek(0)
|
||||
msvcrt.locking(self._lock_handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(self._lock_handle, fcntl.LOCK_UN)
|
||||
self._lock_handle.close()
|
||||
self._lock_handle = None
|
||||
|
||||
async def inspect_template(self, settings):
|
||||
config = settings
|
||||
if config.desired * config.cpu > config.total_cpu or config.desired * config.memory_mb > config.total_memory_mb:
|
||||
raise PoolUnavailable("Worker 总资源超过设置的预算,请减少数量或调整预算")
|
||||
if config.desired * (config.database_pool_size + 2 * config.postgres_pool_size) > config.database_connection_budget:
|
||||
raise PoolUnavailable("Worker 数据库连接总额度超过预算,请减少数量或单节点连接数")
|
||||
if not config.source_container or not config.network:
|
||||
raise PoolUnavailable("请填写源容器名称和 Docker 网络")
|
||||
engine = DockerEngine(config)
|
||||
source = await asyncio.to_thread(engine.request, "GET", f"/containers/{quote(config.source_container, safe='')}/json?size=true")
|
||||
if source.get("Config", {}).get("Labels", {}).get("deerflow.worker-pool"):
|
||||
raise PoolUnavailable("源容器必须是主服务,不能选择已有 Worker")
|
||||
backend = await asyncio.to_thread(
|
||||
engine.configured_storage_backend, source, config.python_executable, config.working_dir,
|
||||
)
|
||||
probe_key = (
|
||||
config.transport, config.host, config.port, config.username, config.source_container,
|
||||
source.get("Id"), source.get("Image"), config.python_executable,
|
||||
config.working_dir, source.get("Config", {}).get("User", ""), backend,
|
||||
)
|
||||
if probe_key == self._python_probe_key and self._python_probe_result:
|
||||
python_executable = self._python_probe_result
|
||||
else:
|
||||
python_executable = await asyncio.to_thread(
|
||||
engine.discover_worker_python, source, config.python_executable, config.working_dir,
|
||||
)
|
||||
self._python_probe_key = probe_key
|
||||
self._python_probe_result = python_executable
|
||||
if python_executable != config.python_executable:
|
||||
config = config.model_copy(update={"python_executable": python_executable})
|
||||
self.store.save(config)
|
||||
self._python_probe_key = (
|
||||
config.transport, config.host, config.port, config.username, config.source_container,
|
||||
source.get("Id"), source.get("Image"), config.python_executable,
|
||||
config.working_dir, source.get("Config", {}).get("User", ""), backend,
|
||||
)
|
||||
runtime_image = None
|
||||
if config.enabled and config.desired > 0:
|
||||
image_key = (source.get("Id"), source.get("Image"), python_executable, backend)
|
||||
if image_key != self._worker_image_key or not self._worker_image_result:
|
||||
self._worker_image_result = await asyncio.to_thread(
|
||||
engine.snapshot_worker_runtime, source, self.store.raw()["instance"],
|
||||
)
|
||||
self._worker_image_key = image_key
|
||||
runtime_image = self._worker_image_result
|
||||
spec = worker_spec(
|
||||
source, config, self.store.raw()["instance"], self.store.raw()["token"],
|
||||
str(self.store.directory), runtime_image=runtime_image,
|
||||
require_shared_archive=backend == "postgres" and config.enabled and config.desired > 0,
|
||||
)
|
||||
return engine, spec
|
||||
|
||||
async def reconcile(self):
|
||||
async with self.lock:
|
||||
settings = self.store.load()
|
||||
if not settings.enabled and not settings.source_container:
|
||||
return
|
||||
db = self._ensure_db()
|
||||
engine, spec = await self.inspect_template(settings)
|
||||
instance = self.store.raw()["instance"]
|
||||
filters = quote(json.dumps({"label": [f"deerflow.worker-pool={instance}"]}))
|
||||
containers = await asyncio.to_thread(engine.request, "GET", f"/containers/json?all=1&filters={filters}")
|
||||
self.discovery_complete = True
|
||||
desired = settings.desired if settings.enabled else 0
|
||||
wanted = {f"dfw-{instance}-{i}" for i in range(desired)}
|
||||
existing = {row["Names"][0].lstrip("/"): row for row in containers}
|
||||
stale_templates = set()
|
||||
for name, row in list(existing.items()):
|
||||
self.nodes.setdefault(name, {"name": name, "id": row["Id"], "ready": False, "active": 0, "draining": name not in wanted, "url": f"http://{name}:8001"})
|
||||
if name in wanted and row.get("State") == "running" and row.get("ImageID") and row["ImageID"] != spec["Image"]:
|
||||
stale_templates.add(name)
|
||||
self.nodes[name]["draining"] = True
|
||||
|
||||
# A stopped container cannot be executing a request. If its saved
|
||||
# startup template has an old Python path, replace it with the
|
||||
# resolved template instead of restarting the same broken command.
|
||||
for name, row in list(existing.items()):
|
||||
if name not in wanted or row["State"] == "running":
|
||||
continue
|
||||
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{quote(row['Id'], safe='')}/json")
|
||||
worker_config = detail.get("Config", {})
|
||||
stale_template = (
|
||||
detail.get("Image") != spec["Image"]
|
||||
or worker_config.get("Entrypoint") != spec["Entrypoint"]
|
||||
or worker_config.get("WorkingDir") != spec["WorkingDir"]
|
||||
)
|
||||
if not stale_template:
|
||||
continue
|
||||
if hasattr(self, "diagnostics"):
|
||||
await self.diagnostics.archive_logs(engine, name)
|
||||
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{quote(row['Id'], safe='')}")
|
||||
db.execute("DELETE FROM affinity WHERE node=?", (name,))
|
||||
db.commit()
|
||||
existing.pop(name, None)
|
||||
self.nodes.pop(name, None)
|
||||
for name, row in list(existing.items()):
|
||||
previous = self.nodes.get(name, {})
|
||||
node = {
|
||||
"name": name,
|
||||
"id": row["Id"],
|
||||
"ready": previous.get("ready", False),
|
||||
"ever_ready": previous.get("ever_ready", False),
|
||||
"active": previous.get("active", 0),
|
||||
"capacity": previous.get("capacity"),
|
||||
"draining": name not in wanted or name in stale_templates,
|
||||
"url": f"http://{name}:8001",
|
||||
"checked_at": time.time(),
|
||||
}
|
||||
for key in ("running", "waiting", "process_started_at", "runtime_revision", "event_loop_lag_ms", "http"):
|
||||
if key in previous:
|
||||
node[key] = previous[key]
|
||||
self.nodes[name] = node
|
||||
if row["State"] != "running":
|
||||
node["ready"] = False
|
||||
node["ever_ready"] = False
|
||||
# A confirmed stopped process cannot keep executing. Old
|
||||
# conversations can move only after this confirmation.
|
||||
db.execute("DELETE FROM affinity WHERE node=?", (name,))
|
||||
db.commit()
|
||||
if name in wanted:
|
||||
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/start")
|
||||
else:
|
||||
if hasattr(self, "diagnostics"):
|
||||
await self.diagnostics.archive_logs(engine, name)
|
||||
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{row['Id']}")
|
||||
self.nodes.pop(name, None)
|
||||
continue
|
||||
try:
|
||||
diagnostics_enabled = bool(getattr(getattr(self, "diagnostics", None), "enabled", False))
|
||||
response = await self.http.get(
|
||||
node["url"] + "/internal/worker-status",
|
||||
headers={
|
||||
"x-deerflow-worker-token": self.store.raw()["token"],
|
||||
"x-deerflow-diagnostics-enabled": "true" if diagnostics_enabled else "false",
|
||||
},
|
||||
timeout=3,
|
||||
)
|
||||
response.raise_for_status()
|
||||
status = response.json()
|
||||
node.update(ready=status["ready"], active=status["active"], capacity=status["capacity"], ever_ready=True)
|
||||
for key in ("running", "waiting", "process_started_at", "runtime_revision", "event_loop_lag_ms", "http"):
|
||||
if key in status:
|
||||
node[key] = status[key]
|
||||
except (httpx.HTTPError, ValueError, KeyError):
|
||||
node["ready"] = False
|
||||
node["error"] = "等待就绪或连接失败;请检查主容器是否位于同一网络、镜像代码和聊天存储配置"
|
||||
continue
|
||||
if node["draining"] and not node["active"] and not self.inflight.get(name, 0):
|
||||
# Refuse all new reservations while stop is in flight.
|
||||
node["ready"] = False
|
||||
node["ever_ready"] = False
|
||||
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/stop?t=30")
|
||||
if hasattr(self, "diagnostics"):
|
||||
await self.diagnostics.archive_logs(engine, name)
|
||||
await asyncio.to_thread(engine.request, "DELETE", f"/containers/{row['Id']}")
|
||||
db.execute("DELETE FROM affinity WHERE node=?", (name,))
|
||||
db.commit()
|
||||
self.nodes.pop(name, None)
|
||||
existing.pop(name, None)
|
||||
# Replacements are created only after the previous container has
|
||||
# drained and been removed. This also refreshes Workers after the
|
||||
# main service's runtime image or configured storage backend changes.
|
||||
for name in sorted(wanted - existing.keys()):
|
||||
row = await asyncio.to_thread(engine.request, "POST", f"/containers/create?name={name}", spec)
|
||||
self.nodes[name] = {
|
||||
"name": name,
|
||||
"id": row["Id"],
|
||||
"ready": False,
|
||||
"active": 0,
|
||||
"draining": False,
|
||||
"url": f"http://{name}:8001",
|
||||
}
|
||||
await asyncio.to_thread(engine.request, "POST", f"/containers/{row['Id']}/start")
|
||||
existing[name] = {"Id": row["Id"], "State": "running", "ImageID": spec["Image"]}
|
||||
# Missing nodes are not silently reassigned: an operator must
|
||||
# reconcile the actual container state before changing affinity.
|
||||
self.error = None
|
||||
|
||||
async def reserve(self, thread: str, *, create: bool) -> dict | None:
|
||||
settings = self.store.load()
|
||||
if not settings.enabled and not self.nodes:
|
||||
return None
|
||||
db = self._ensure_db()
|
||||
# Container creation/SSH can be slow; routing must never wait on that lock.
|
||||
async with self.route_lock:
|
||||
row = db.execute("SELECT node FROM affinity WHERE thread=?", (thread,)).fetchone()
|
||||
if row is None and create and self.local_threads.get(thread, 0):
|
||||
raise PoolUnavailable("该会话正在主服务执行,请等待当前轮完成")
|
||||
if row:
|
||||
node = self.nodes.get(row[0])
|
||||
if node and node.get("draining") and create:
|
||||
return {"_fallback_to_main": True, "_fallback_reason": "worker_draining", "_worker": dict(node)}
|
||||
# A delayed health probe stops new work, but must not hide an
|
||||
# existing run's result. Keep reads/cancellation on its owner;
|
||||
# stopped containers clear ever_ready before affinity changes.
|
||||
if not node or (not node["ready"] and (create or not node.get("ever_ready"))):
|
||||
raise PoolUnavailable("该会话的执行节点暂不可用,请稍后重试;系统不会重复提交任务")
|
||||
if node["draining"] and create:
|
||||
raise PoolUnavailable("该会话节点正在停止接单,请等待当前任务完成后重试")
|
||||
if create:
|
||||
capacity = max(1, int(node.get("capacity") or settings.concurrency))
|
||||
load = max(int(node.get("active") or 0), self.inflight.get(node["name"], 0))
|
||||
if load >= capacity:
|
||||
return {"_fallback_to_main": True, "_fallback_reason": "worker_saturated", "_worker": dict(node)}
|
||||
elif not settings.enabled or not create:
|
||||
return None
|
||||
else:
|
||||
choices = [n for n in self.nodes.values() if n["ready"] and not n["draining"]]
|
||||
if not choices:
|
||||
return {"_fallback_to_main": True, "_fallback_reason": "workers_unavailable"}
|
||||
available = []
|
||||
for candidate in choices:
|
||||
capacity = max(1, int(candidate.get("capacity") or settings.concurrency))
|
||||
load = max(int(candidate.get("active") or 0), self.inflight.get(candidate["name"], 0))
|
||||
if load < capacity:
|
||||
available.append((load, candidate))
|
||||
if not available:
|
||||
return {"_fallback_to_main": True, "_fallback_reason": "worker_pool_saturated"}
|
||||
_, node = min(available, key=lambda item: (item[0], item[1]["name"]))
|
||||
db.execute("INSERT INTO affinity VALUES (?, ?)", (thread, node["name"]))
|
||||
db.commit()
|
||||
self.inflight[node["name"]] = self.inflight.get(node["name"], 0) + 1
|
||||
return dict(node)
|
||||
|
||||
def release(self, name):
|
||||
self.inflight[name] = max(0, self.inflight.get(name, 0) - 1)
|
||||
|
||||
def status(self):
|
||||
return {"settings": self.store.public(), "nodes": [{k: v for k, v in n.items() if k != "url"} for n in self.nodes.values()], "error": self.error}
|
||||
@@ -0,0 +1,348 @@
|
||||
"""Admin-enabled, bounded diagnostics with independent, short DB probes."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import deque
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from .docker import DockerEngine, DockerError, decode_docker_output
|
||||
from .host_metrics import rates
|
||||
from .telemetry import RequestRecorder
|
||||
|
||||
|
||||
def redact_logs(value):
|
||||
value = re.sub(r"(?i)([a-z][a-z0-9+.-]*://)[^\s/@]+:[^\s/@]+@", r"\1[REDACTED]@", value)
|
||||
value = re.sub(r"(?i)(Bearer\s+)[^\s\"']+", r"\1[REDACTED]", value)
|
||||
value = re.sub(r"(?i)((?:password|passwd|api[_-]?key|access[_-]?token|secret|authorization|token)[\"']?\s*[:=]\s*[\"']?)[^\s,\"'}]+", r"\1[REDACTED]", value)
|
||||
value = re.sub(r"\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+", "[REDACTED_JWT]", value)
|
||||
value = re.sub(r"\bsk-[a-zA-Z0-9_-]{12,}", "[REDACTED_KEY]", value)
|
||||
return value
|
||||
|
||||
|
||||
def container_metrics(detail, stats, previous_stats=None):
|
||||
cpu = stats.get("cpu_stats", {})
|
||||
previous = previous_stats.get("cpu_stats", {}) if previous_stats else stats.get("precpu_stats", {})
|
||||
delta = cpu.get("cpu_usage", {}).get("total_usage", 0) - previous.get("cpu_usage", {}).get("total_usage", 0)
|
||||
system_delta = cpu.get("system_cpu_usage", 0) - previous.get("system_cpu_usage", 0)
|
||||
cores = cpu.get("online_cpus") or len(cpu.get("cpu_usage", {}).get("percpu_usage", [])) or 1
|
||||
memory = stats.get("memory_stats", {})
|
||||
cache = memory.get("stats", {}).get("inactive_file", memory.get("stats", {}).get("total_inactive_file", 0))
|
||||
state = detail.get("State", {})
|
||||
return {
|
||||
"name": detail.get("Name", "").lstrip("/"),
|
||||
"image": detail.get("Config", {}).get("Image", ""),
|
||||
"image_id": detail.get("Image", "")[:22],
|
||||
"created_at": detail.get("Created"),
|
||||
"started_at": state.get("StartedAt"),
|
||||
"status": state.get("Status"),
|
||||
"health": state.get("Health", {}).get("Status"),
|
||||
"restarts": detail.get("RestartCount", 0),
|
||||
"oom_killed": state.get("OOMKilled", False),
|
||||
"cpu_percent": round(delta / system_delta * cores * 100, 2) if previous.get("system_cpu_usage") and system_delta > 0 and delta >= 0 else None,
|
||||
"cpu_limit": detail.get("HostConfig", {}).get("NanoCpus", 0) / 1e9 or None,
|
||||
"memory_bytes": max(0, memory.get("usage", 0) - cache),
|
||||
"memory_limit": memory.get("limit"),
|
||||
"pids": stats.get("pids_stats", {}).get("current"),
|
||||
"log_policy": detail.get("HostConfig", {}).get("LogConfig", {}),
|
||||
"host_proc_mounted": any(m.get("Destination") == "/host/proc" for m in detail.get("Mounts", [])),
|
||||
"docker_socket_mounted": any(m.get("Destination") == "/var/run/docker.sock" for m in detail.get("Mounts", [])),
|
||||
}
|
||||
|
||||
|
||||
class Diagnostics:
|
||||
def __init__(self, pool, app):
|
||||
self.pool = pool
|
||||
self.app = app
|
||||
self.path = pool.store.directory / "diagnostics-settings.json"
|
||||
self.archive_directory = pool.store.directory / "log-archive"
|
||||
self.enabled = json.loads(self.path.read_text()).get("enabled", False) if self.path.exists() else False
|
||||
self.recorder = RequestRecorder()
|
||||
self.recorder.enabled = self.enabled
|
||||
app.state.request_recorder = self.recorder
|
||||
self.history = deque(maxlen=720)
|
||||
self.latest = None
|
||||
self.previous = None
|
||||
self.container_previous = {}
|
||||
from .revision import code_revision
|
||||
|
||||
self.runtime_revision = code_revision()
|
||||
self.task = None
|
||||
self.lag_task = None
|
||||
self.loop_lag_ms = 0
|
||||
self.db_engines = {}
|
||||
self.db = None
|
||||
if self.enabled:
|
||||
self._ensure_db()
|
||||
|
||||
def _ensure_db(self):
|
||||
if self.db is None:
|
||||
self.db = sqlite3.connect(self.pool.store.directory / "diagnostics.sqlite3")
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS snapshots (id INTEGER PRIMARY KEY, at REAL, body TEXT)")
|
||||
self.db.execute("CREATE TABLE IF NOT EXISTS requests (id INTEGER PRIMARY KEY, at REAL, body TEXT)")
|
||||
for (value,) in reversed(self.db.execute("SELECT body FROM snapshots ORDER BY id DESC LIMIT 720").fetchall()):
|
||||
self.history.append(json.loads(value))
|
||||
for (value,) in reversed(self.db.execute("SELECT body FROM requests ORDER BY id DESC LIMIT 2000").fetchall()):
|
||||
self.recorder.completed.append(json.loads(value))
|
||||
return self.db
|
||||
|
||||
def start(self):
|
||||
if not self.enabled:
|
||||
return
|
||||
self._ensure_db()
|
||||
if self.task is not None and not self.task.done():
|
||||
return
|
||||
self.task = asyncio.create_task(self.loop())
|
||||
self.lag_task = asyncio.create_task(self.measure_lag())
|
||||
|
||||
async def close(self):
|
||||
for task in (self.task, self.lag_task):
|
||||
if task:
|
||||
task.cancel()
|
||||
await asyncio.gather(*(t for t in (self.task, self.lag_task) if t), return_exceptions=True)
|
||||
self.persist_requests()
|
||||
if self.db is not None:
|
||||
self.db.close()
|
||||
self.db = None
|
||||
for engine in self.db_engines.values():
|
||||
await engine.dispose()
|
||||
|
||||
async def set_enabled(self, enabled):
|
||||
self.enabled = enabled
|
||||
self.recorder.enabled = enabled
|
||||
temporary = self.path.with_suffix(".tmp")
|
||||
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump({"enabled": enabled}, stream)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, self.path)
|
||||
if enabled:
|
||||
self._ensure_db()
|
||||
self.start()
|
||||
else:
|
||||
tasks = [task for task in (self.task, self.lag_task) if task]
|
||||
for task in tasks:
|
||||
if task:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
self.task = None
|
||||
self.lag_task = None
|
||||
self.previous = None
|
||||
self.persist_requests()
|
||||
|
||||
async def measure_lag(self):
|
||||
while True:
|
||||
before = asyncio.get_running_loop().time()
|
||||
await asyncio.sleep(1)
|
||||
self.loop_lag_ms = round(max(0, asyncio.get_running_loop().time() - before - 1) * 1000, 2) if self.enabled else 0
|
||||
|
||||
async def loop(self):
|
||||
while True:
|
||||
if self.enabled:
|
||||
try:
|
||||
await self.sample()
|
||||
except Exception as exc:
|
||||
self.latest = {"at": time.time(), "error": f"采集失败({type(exc).__name__})"}
|
||||
await asyncio.sleep(5)
|
||||
|
||||
def persist_requests(self):
|
||||
batch = []
|
||||
while self.recorder.pending:
|
||||
row = self.recorder.pending.popleft()
|
||||
batch.append((row["started"], json.dumps(row)))
|
||||
if batch and self.db is not None:
|
||||
self.db.executemany("INSERT INTO requests(at, body) VALUES (?, ?)", batch)
|
||||
self.db.execute("DELETE FROM requests WHERE id <= (SELECT COALESCE(MAX(id),0)-10000 FROM requests)")
|
||||
self.db.commit()
|
||||
|
||||
async def sample(self):
|
||||
self._ensure_db()
|
||||
settings = self.pool.store.load()
|
||||
engine = DockerEngine(settings)
|
||||
names = [settings.source_container, *list(self.pool.nodes)]
|
||||
|
||||
async def read_container(name):
|
||||
try:
|
||||
encoded = quote(name, safe="")
|
||||
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/json")
|
||||
stats = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/stats?stream=false&one-shot=true")
|
||||
key = detail.get("Id", name)
|
||||
result = container_metrics(detail, stats, self.container_previous.get(key))
|
||||
self.container_previous[key] = stats
|
||||
return result
|
||||
except Exception as exc:
|
||||
return {"name": name, "error": f"节点指标不可用({type(exc).__name__})"}
|
||||
|
||||
async def host():
|
||||
try:
|
||||
raw = await asyncio.to_thread(engine.host_snapshot, settings.source_container)
|
||||
result = rates(raw, self.previous)
|
||||
self.previous = raw
|
||||
return result
|
||||
except Exception as exc:
|
||||
detail = str(exc) if isinstance(exc, DockerError) else type(exc).__name__
|
||||
return {"available": False, "message": f"宿主机采集不可用:{detail}。请检查只读挂载和 Docker 连接"}
|
||||
|
||||
host_data, containers, databases = await asyncio.gather(host(), asyncio.gather(*(read_container(n) for n in names if n)), self.database_metrics())
|
||||
result = {"at": time.time(), "host": host_data, "containers": containers, "databases": databases, "event_loop_lag_ms": self.loop_lag_ms, "nodes": self.pool.status()["nodes"], "runtime_revision": self.runtime_revision}
|
||||
self.latest = result
|
||||
summary = {
|
||||
"at": result["at"],
|
||||
"cpu_percent": host_data.get("cpu_percent"),
|
||||
"iowait_percent": host_data.get("iowait_percent"),
|
||||
"memory_used_percent": round(100 * (1 - host_data["memory_available"] / host_data["memory_total"]), 2) if host_data.get("memory_total") else None,
|
||||
"rx_bps": sum(v.get("rx_bps") or 0 for v in host_data.get("interfaces", {}).values()),
|
||||
"tx_bps": sum(v.get("tx_bps") or 0 for v in host_data.get("interfaces", {}).values()),
|
||||
"loop_lag_ms": self.loop_lag_ms,
|
||||
"mysql_connections": databases.get("mysql", {}).get("connected"),
|
||||
"postgres_connections": databases.get("postgres", {}).get("connected"),
|
||||
"p95_headers_ms": self.recorder.summary()["p95_headers_ms"],
|
||||
}
|
||||
self.history.append(summary)
|
||||
self.db.execute("INSERT INTO snapshots(at,body) VALUES (?,?)", (result["at"], json.dumps(summary)))
|
||||
self.db.execute("DELETE FROM snapshots WHERE id <= (SELECT COALESCE(MAX(id),0)-720 FROM snapshots)")
|
||||
self.db.commit()
|
||||
self.persist_requests()
|
||||
|
||||
def status(self):
|
||||
return {"enabled": self.enabled, "sample_interval_seconds": 5, "retained_samples": 720, "latest": self.latest, "history": list(self.history), "http": self.recorder.summary()}
|
||||
|
||||
async def database_metrics(self):
|
||||
config = getattr(self.app.state, "config", None)
|
||||
db = getattr(config, "database", None)
|
||||
checkpoint = getattr(config, "checkpointer", None)
|
||||
result = {}
|
||||
if getattr(db, "backend", None) == "mysql":
|
||||
result["mysql"] = await self.mysql_metrics(db.app_sqlalchemy_url)
|
||||
else:
|
||||
result["mysql"] = {"available": False, "reason": "业务库未使用 MySQL"}
|
||||
from .preflight import postgres_connection_string
|
||||
|
||||
pg_url = postgres_connection_string(config)
|
||||
if pg_url:
|
||||
result["postgres"] = await self.postgres_metrics(pg_url)
|
||||
else:
|
||||
result["postgres"] = {"available": False, "configured": False, "reason": "当前聊天存储为 SQLite,未启用 PostgreSQL,不能显示为 0 个连接"}
|
||||
result["chat_backend"] = "postgres" if pg_url else ("sqlite" if checkpoint and checkpoint.type in {"sqlite", "postgres"} else "unknown")
|
||||
return result
|
||||
|
||||
async def mysql_metrics(self, url):
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
key = "mysql"
|
||||
try:
|
||||
if key not in self.db_engines:
|
||||
self.db_engines[key] = create_async_engine(url, pool_size=1, max_overflow=0, pool_timeout=1, pool_pre_ping=False, connect_args={"connect_timeout": 2})
|
||||
async with asyncio.timeout(4):
|
||||
async with self.db_engines[key].connect() as connection:
|
||||
statuses = dict((await connection.execute(text("SHOW GLOBAL STATUS WHERE Variable_name IN ('Threads_connected','Threads_running','Max_used_connections','Slow_queries','Aborted_connects')"))).all())
|
||||
variables = dict((await connection.execute(text("SHOW GLOBAL VARIABLES WHERE Variable_name IN ('max_connections','long_query_time')"))).all())
|
||||
rows = (await connection.execute(text("SHOW PROCESSLIST"))).mappings().all()
|
||||
running = [r for r in rows if str(r.get("Command", "")).lower() not in {"sleep", "daemon"}]
|
||||
return {
|
||||
"available": True,
|
||||
"connected": int(statuses.get("Threads_connected", 0)),
|
||||
"running": int(statuses.get("Threads_running", 0)),
|
||||
"max_connections": int(variables.get("max_connections", 0)),
|
||||
"peak_connections": int(statuses.get("Max_used_connections", 0)),
|
||||
"slow_queries": int(statuses.get("Slow_queries", 0)),
|
||||
"slow_query_threshold_seconds": float(variables.get("long_query_time", 0)),
|
||||
"longest_active_seconds": max((int(r.get("Time", 0)) for r in running), default=0),
|
||||
"processes_visible": len(rows),
|
||||
"note": "连接总数为 MySQL 全局值(包含采集连接);最长执行时间只覆盖当前账号可见进程,不读取 SQL 文本。",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"available": False, "error": f"MySQL 指标不可用({type(exc).__name__}),请检查连接与只读监控权限"}
|
||||
|
||||
async def postgres_metrics(self, url):
|
||||
try:
|
||||
import psycopg
|
||||
|
||||
async with asyncio.timeout(4):
|
||||
async with await psycopg.AsyncConnection.connect(url, connect_timeout=2, autocommit=True) as connection:
|
||||
async with connection.cursor() as cursor:
|
||||
query = (
|
||||
"SELECT count(*), count(*) FILTER (WHERE state='active'), "
|
||||
"count(*) FILTER (WHERE wait_event_type='Lock'), "
|
||||
"COALESCE(max(EXTRACT(EPOCH FROM now()-query_start)) "
|
||||
"FILTER (WHERE state='active' AND pid<>pg_backend_pid()), 0) "
|
||||
"FROM pg_stat_activity"
|
||||
)
|
||||
await cursor.execute(query)
|
||||
connected, active, locks, longest = await cursor.fetchone()
|
||||
await cursor.execute("SHOW max_connections")
|
||||
maximum = (await cursor.fetchone())[0]
|
||||
return {
|
||||
"available": True,
|
||||
"configured": True,
|
||||
"connected": connected,
|
||||
"active": active,
|
||||
"lock_waits": locks,
|
||||
"max_connections": int(maximum),
|
||||
"longest_active_seconds": float(longest),
|
||||
"note": "按 pg_stat_activity 采集,普通账号可能看不到其他账号的状态和耗时;包含采集连接,不读取 SQL 文本。",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"available": False, "configured": True, "error": f"PostgreSQL 指标不可用({type(exc).__name__}),请检查连接与监控权限"}
|
||||
|
||||
async def logs(self, name, tail=200, minutes=60, query=""):
|
||||
if name.startswith("archive:"):
|
||||
return self.read_archive(name[len("archive:") :], query)
|
||||
settings = self.pool.store.load()
|
||||
if name != settings.source_container and name not in self.pool.nodes:
|
||||
raise DockerError("只能查看当前主服务和本 Worker 池的日志")
|
||||
engine = DockerEngine(settings)
|
||||
encoded = quote(name, safe="")
|
||||
detail = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/json")
|
||||
if name != settings.source_container and detail.get("Config", {}).get("Labels", {}).get("deerflow.worker-pool") != self.pool.store.raw()["instance"]:
|
||||
raise DockerError("节点不属于当前 Worker 池")
|
||||
params = urlencode({"stdout": 1, "stderr": 1, "timestamps": 1, "tail": min(1000, max(1, tail)), "since": int(time.time() - min(1440, max(1, minutes)) * 60)})
|
||||
data = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/logs?{params}", raw=True, max_bytes=1024 * 1024)
|
||||
lines = redact_logs(decode_docker_output(data)).splitlines()
|
||||
if query:
|
||||
lines = [line for line in lines if query.casefold() in line.casefold()]
|
||||
return {"node": name, "lines": lines[-1000:], "log_policy": detail.get("HostConfig", {}).get("LogConfig", {}), "limited_bytes": 1024 * 1024}
|
||||
|
||||
async def archive_logs(self, engine, name):
|
||||
"""Keep a bounded redacted tail BEFORE removing an owned container."""
|
||||
import gzip
|
||||
|
||||
encoded = quote(name, safe="")
|
||||
params = urlencode({"stdout": 1, "stderr": 1, "timestamps": 1, "tail": 10000})
|
||||
data = await asyncio.to_thread(engine.request, "GET", f"/containers/{encoded}/logs?{params}", raw=True, max_bytes=2 * 1024 * 1024)
|
||||
archive_id = f"{name}-{time.time_ns()}"
|
||||
payload = {"node": name, "created_at": time.time(), "lines": redact_logs(decode_docker_output(data)).splitlines(), "limited_bytes": 2 * 1024 * 1024, "note": "节点删除前的最近 10000 行 / 最多 2 MB 归档;保存 7 天,最多 50 份。"}
|
||||
self.archive_directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
target = self.archive_directory / f"{archive_id}.json.gz"
|
||||
fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
out.write(gzip.compress(json.dumps(payload).encode()))
|
||||
files = sorted(self.archive_directory.glob("*.json.gz"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
for index, path in enumerate(files):
|
||||
if index >= 50 or path.stat().st_mtime < time.time() - 7 * 86400:
|
||||
path.unlink()
|
||||
|
||||
def archives(self):
|
||||
return [
|
||||
{"id": p.name.removesuffix(".json.gz"), "name": p.name.removesuffix(".json.gz"), "created_at": p.stat().st_mtime}
|
||||
for p in sorted(self.archive_directory.glob("*.json.gz"), key=lambda p: p.stat().st_mtime, reverse=True)[:50]
|
||||
if p.stat().st_mtime >= time.time() - 7 * 86400
|
||||
]
|
||||
|
||||
def read_archive(self, archive_id, query=""):
|
||||
import gzip
|
||||
|
||||
if not re.fullmatch(r"[A-Za-z0-9_.-]{1,180}", archive_id):
|
||||
raise DockerError("无效日志归档")
|
||||
path = self.archive_directory / f"{archive_id}.json.gz"
|
||||
if not path.is_file() or path.stat().st_mtime < time.time() - 7 * 86400:
|
||||
raise DockerError("日志归档不存在或已过期")
|
||||
payload = json.loads(gzip.decompress(path.read_bytes()))
|
||||
payload["lines"] = [line for line in payload["lines"] if query.casefold() in line.casefold()][-1000:]
|
||||
return payload
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Read-only server discovery and conservative starter recommendations."""
|
||||
import base64
|
||||
import hashlib
|
||||
import math
|
||||
import socket
|
||||
|
||||
from .docker import DockerEngine, DockerError
|
||||
|
||||
|
||||
def host_fingerprint(host: str, port: int) -> str:
|
||||
import paramiko
|
||||
transport = None
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=10) as connection:
|
||||
transport = paramiko.Transport(connection)
|
||||
transport.start_client(timeout=10)
|
||||
key = transport.get_remote_server_key()
|
||||
return "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
|
||||
except Exception as exc:
|
||||
raise DockerError(f"读取服务器指纹失败({type(exc).__name__})") from None
|
||||
finally:
|
||||
if transport is not None:
|
||||
transport.close()
|
||||
|
||||
|
||||
def recommend(cpus, memory_mb, reserved_cpu=0, reserved_memory_mb=0, connection_budget=64):
|
||||
# Reserve at least half the host for the main app, DB and other services.
|
||||
cpu_budget = max(0, min(cpus / 2, cpus - reserved_cpu - 1))
|
||||
memory_budget = max(0, min(memory_mb // 2, memory_mb - reserved_memory_mb - 2048))
|
||||
cpu = 2 if cpu_budget >= 2 else 1
|
||||
memory = 2048
|
||||
count = max(0, min(4, math.floor(cpu_budget / cpu), memory_budget // memory, connection_budget // 12))
|
||||
return {"desired": int(count), "cpu": cpu, "memory_mb": memory,
|
||||
"total_cpu": max(cpu, count * cpu), "total_memory_mb": max(memory, count * memory),
|
||||
"concurrency": 4, "database_pool_size": 4, "postgres_pool_size": 4}
|
||||
|
||||
|
||||
def discover(settings, instance):
|
||||
engine = DockerEngine(settings)
|
||||
info = engine.request("GET", "/info")
|
||||
containers = engine.request("GET", "/containers/json")
|
||||
cpu_reserved = 0
|
||||
memory_reserved = 0
|
||||
templates = []
|
||||
for item in containers:
|
||||
if item.get("Labels", {}).get("deerflow.worker-pool") == instance:
|
||||
continue
|
||||
detail = engine.request("GET", f"/containers/{item['Id']}/json")
|
||||
limits = detail.get("HostConfig", {})
|
||||
cpu_reserved += limits.get("NanoCpus", 0) / 1_000_000_000
|
||||
memory_reserved += limits.get("Memory", 0) // (1024 ** 2)
|
||||
if item.get("Labels", {}).get("deerflow.worker-role") == "online":
|
||||
continue
|
||||
networks = [name for name in detail.get("NetworkSettings", {}).get("Networks", {}) if name not in {"host", "none", "bridge"}]
|
||||
templates.append({"name": detail.get("Name", "").lstrip("/"), "networks": networks})
|
||||
cpus = int(info.get("NCPU", 0))
|
||||
memory_mb = int(info.get("MemTotal", 0)) // (1024 ** 2)
|
||||
return {"cpus": cpus, "memory_mb": memory_mb, "containers": templates,
|
||||
"recommendation": recommend(cpus, memory_mb, cpu_reserved, memory_reserved, settings.database_connection_budget),
|
||||
"note": "按硬件总量和其他容器已配置的资源上限估算,至少为主服务、数据库及系统保留一半资源;不是实时空闲内存。建议先用最多 4 个节点,再按实际负载调整。"}
|
||||
@@ -0,0 +1,505 @@
|
||||
"""Docker Engine API over a Unix socket or a pinned SSH connection.
|
||||
|
||||
Only the constant `docker system dial-stdio` command runs on the server.
|
||||
All variable data travels as JSON, never interpolated into a shell command.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import posixpath
|
||||
import socket
|
||||
from urllib.parse import quote
|
||||
|
||||
from .settings import PoolSettings
|
||||
|
||||
|
||||
class DockerError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
_WORKER_STORAGE_SELECTION = """\
|
||||
import os
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
config_path = Path(os.getenv('DEER_FLOW_CONFIG_PATH') or 'config.yaml')
|
||||
config = yaml.safe_load(config_path.read_text(encoding='utf-8')) or {}
|
||||
checkpoint = config.get('checkpointer')
|
||||
if checkpoint is not None:
|
||||
checkpoint_type = checkpoint.get('type')
|
||||
if checkpoint_type == 'postgres':
|
||||
raw = os.getenv('DEER_FLOW_USE_PG_CHECKPOINTER')
|
||||
if raw:
|
||||
postgres_enabled = raw.strip().lower() not in {'0', 'false', 'no', 'off'}
|
||||
else:
|
||||
postgres_enabled = bool(checkpoint.get('use_postgres', True))
|
||||
backend = 'postgres' if postgres_enabled else 'sqlite'
|
||||
elif checkpoint_type == 'sqlite':
|
||||
backend = 'sqlite'
|
||||
else:
|
||||
backend = 'memory'
|
||||
else:
|
||||
backend = (config.get('database') or {}).get('backend', 'memory')
|
||||
if backend not in {'postgres', 'sqlite'}:
|
||||
backend = 'memory'
|
||||
"""
|
||||
_WORKER_STORAGE_PROBE = _WORKER_STORAGE_SELECTION + """\
|
||||
print('__DEERFLOW_STORAGE_BACKEND__=' + backend)
|
||||
"""
|
||||
_WORKER_PYTHON_PROBE = _WORKER_STORAGE_SELECTION + """\
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
|
||||
if sys.version_info < (3, 12):
|
||||
raise SystemExit(78)
|
||||
required = ['uvicorn']
|
||||
if backend == 'postgres':
|
||||
required.extend((
|
||||
'langgraph.checkpoint.postgres.aio',
|
||||
'langgraph.store.postgres.aio',
|
||||
'psycopg.rows',
|
||||
'psycopg_pool',
|
||||
))
|
||||
elif backend == 'sqlite':
|
||||
required.extend((
|
||||
'langgraph.checkpoint.sqlite.aio',
|
||||
'langgraph.store.sqlite.aio',
|
||||
'aiosqlite',
|
||||
))
|
||||
missing = []
|
||||
for module in required:
|
||||
try:
|
||||
if importlib.util.find_spec(module) is None:
|
||||
missing.append(module)
|
||||
except (ImportError, ModuleNotFoundError, ValueError):
|
||||
missing.append(module)
|
||||
if not Path('app/gateway/worker_pool/worker.py').is_file():
|
||||
missing.append('app.gateway.worker_pool.worker')
|
||||
if missing:
|
||||
packages = (
|
||||
'langgraph-checkpoint-postgres psycopg[binary] psycopg-pool'
|
||||
if backend == 'postgres'
|
||||
else 'langgraph-checkpoint-sqlite aiosqlite'
|
||||
)
|
||||
print('__DEERFLOW_STORAGE_ERROR__=Worker Python is missing configured runtime modules: '
|
||||
+ ', '.join(missing) + '; install ' + packages, flush=True)
|
||||
raise SystemExit(78)
|
||||
print('__DEERFLOW_WORKER_PYTHON__=' + os.path.abspath(sys.executable))
|
||||
"""
|
||||
_WORKER_PYTHON_PROBE_SHELL = r'''
|
||||
probe() {
|
||||
candidate="$1"
|
||||
[ -n "$candidate" ] || return 1
|
||||
output=$("$candidate" -c "$2" 2>/dev/null) || {
|
||||
case "$output" in
|
||||
*__DEERFLOW_STORAGE_ERROR__=*)
|
||||
error=${output##*__DEERFLOW_STORAGE_ERROR__=}
|
||||
printf '__DEERFLOW_STORAGE_ERROR__=%s\n' "$error"
|
||||
;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
case "$output" in
|
||||
*__DEERFLOW_WORKER_PYTHON__=*)
|
||||
resolved=${output##*__DEERFLOW_WORKER_PYTHON__=}
|
||||
case "$resolved" in /*) printf '%s\n' "$resolved"; return 0 ;; esac
|
||||
;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
code="$1"
|
||||
shift
|
||||
for candidate in "$@"; do
|
||||
if probe "$candidate" "$code"; then exit 0; fi
|
||||
done
|
||||
# The main API may use a Python executable outside the image's default PATH.
|
||||
# Probe the interpreter actually running in this source container before PATH
|
||||
# fallbacks, using the same configured persistence-adapter check above.
|
||||
for process_exe in /proc/[0-9]*/exe; do
|
||||
candidate=$(readlink "$process_exe" 2>/dev/null) || continue
|
||||
case "$candidate" in
|
||||
*/python|*/python[0-9]|*/python[0-9].* )
|
||||
case "$candidate" in *" (deleted)"*) continue ;; esac
|
||||
if probe "$candidate" "$code"; then exit 0; fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
for candidate in python3 python3.14 python3.13 python3.12 python3.11 python; do
|
||||
if probe "$candidate" "$code"; then exit 0; fi
|
||||
done
|
||||
exit 1
|
||||
'''
|
||||
|
||||
|
||||
class DockerEngine:
|
||||
def __init__(self, settings: PoolSettings):
|
||||
self.settings = settings
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None, *, raw=False, max_bytes=None, json_stream=False):
|
||||
client = None
|
||||
channel = None
|
||||
# Committing a large production backend layer can take longer than
|
||||
# the ordinary metadata/exec requests. Do not turn a slow but valid
|
||||
# runtime snapshot into a permanent Worker reconciliation failure.
|
||||
request_timeout = 300 if method == "POST" and path.startswith("/commit?") else 20
|
||||
try:
|
||||
if self.settings.transport == "socket":
|
||||
channel = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
channel.settimeout(request_timeout)
|
||||
channel.connect("/var/run/docker.sock")
|
||||
else:
|
||||
import paramiko
|
||||
|
||||
expected = self.settings.host_key_sha256
|
||||
if not expected.startswith("SHA256:"):
|
||||
raise DockerError("请先填写服务器 SSH 主机密钥指纹(SHA256:…)")
|
||||
|
||||
class PinnedKey(paramiko.MissingHostKeyPolicy):
|
||||
def missing_host_key(self, ssh, hostname, key):
|
||||
actual = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
|
||||
if actual != expected:
|
||||
raise DockerError("SSH 主机密钥指纹不匹配,连接已拒绝")
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(PinnedKey())
|
||||
client.connect(
|
||||
self.settings.host,
|
||||
port=self.settings.port,
|
||||
username=self.settings.username,
|
||||
password=self.settings.password,
|
||||
timeout=10,
|
||||
auth_timeout=10,
|
||||
banner_timeout=10,
|
||||
look_for_keys=False,
|
||||
allow_agent=False,
|
||||
)
|
||||
channel = client.get_transport().open_session(timeout=20)
|
||||
channel.settimeout(request_timeout)
|
||||
channel.exec_command("docker system dial-stdio")
|
||||
data = json.dumps(body).encode() if body is not None else b""
|
||||
request = f"{method} {path} HTTP/1.1\r\nHost: docker\r\nContent-Type: application/json\r\nContent-Length: {len(data)}\r\nConnection: close\r\n\r\n".encode() + data
|
||||
channel.sendall(request)
|
||||
response = http.client.HTTPResponse(channel)
|
||||
response.begin()
|
||||
if json_stream and response.status < 400:
|
||||
# Exec uses a hijacked connection that may outlive the process.
|
||||
# Stop on a complete bounded JSON result instead of waiting for EOF.
|
||||
result = b""
|
||||
while len(result) < (max_bytes or 1024 * 1024):
|
||||
chunk = response.read1(min(65536, (max_bytes or 1024 * 1024) - len(result)))
|
||||
if not chunk:
|
||||
break
|
||||
result += chunk
|
||||
try:
|
||||
json.loads(decode_docker_output(result))
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
result = response.read(max_bytes) if max_bytes else response.read()
|
||||
if response.status >= 400:
|
||||
# Docker errors can contain environment values / host paths.
|
||||
raise DockerError(f"Docker 操作失败(HTTP {response.status}),请检查容器、网络和账号权限")
|
||||
return result if raw else (json.loads(result) if result else {})
|
||||
except DockerError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DockerError(f"服务器连接失败({type(exc).__name__}),请检查地址、账号、密码和 Docker 权限") from None
|
||||
finally:
|
||||
if channel is not None:
|
||||
channel.close()
|
||||
if client is not None:
|
||||
client.close()
|
||||
|
||||
def _exec_capture(self, container_id: str, command: list[str], working_dir: str, user: str) -> tuple[int, str]:
|
||||
created = self.request("POST", f"/containers/{quote(container_id, safe='')}/exec", {
|
||||
"AttachStdout": True,
|
||||
"AttachStderr": True,
|
||||
"Cmd": command,
|
||||
"WorkingDir": working_dir,
|
||||
"User": user,
|
||||
})
|
||||
exec_id = quote(created["Id"], safe="")
|
||||
output = self.request(
|
||||
"POST", f"/exec/{exec_id}/start", {"Detach": False, "Tty": False},
|
||||
raw=True, max_bytes=1024 * 1024,
|
||||
)
|
||||
inspected = self.request("GET", f"/exec/{exec_id}/json")
|
||||
exit_code = inspected.get("ExitCode")
|
||||
return (int(exit_code) if exit_code is not None else 1), decode_docker_output(output)
|
||||
|
||||
@staticmethod
|
||||
def _python_candidates(source: dict, preferred: str) -> list[str]:
|
||||
candidates = [
|
||||
preferred,
|
||||
"/opt/venv/bin/python", "/opt/venv/bin/python3",
|
||||
"/app/.venv/bin/python", "/app/.venv/bin/python3",
|
||||
"/app/backend/.venv/bin/python", "/venv/bin/python", "/venv/bin/python3",
|
||||
"/usr/local/bin/python", "/usr/local/bin/python3",
|
||||
"/usr/local/bin/python3.14", "/usr/local/bin/python3.13",
|
||||
"/usr/local/bin/python3.12", "/usr/local/bin/python3.11",
|
||||
"/usr/bin/python", "/usr/bin/python3", "/usr/bin/python3.12",
|
||||
]
|
||||
env = source.get("Config", {}).get("Env", [])
|
||||
path_value = next((item[5:] for item in env if item.startswith("PATH=")), "")
|
||||
for directory in path_value.split(":"):
|
||||
if directory.startswith("/"):
|
||||
candidates.extend(f"{directory.rstrip('/')}/{name}" for name in ("python3", "python"))
|
||||
return list(dict.fromkeys(path for path in candidates if path and path.startswith("/")))
|
||||
|
||||
def discover_worker_python(self, source: dict, preferred: str, working_dir: str) -> str:
|
||||
"""Find an interpreter inside the source image that can actually start a Worker."""
|
||||
container_id = source.get("Id")
|
||||
if not container_id:
|
||||
raise DockerError("无法读取主后端容器 ID,不能检测镜像内 Python")
|
||||
config = source.get("Config", {})
|
||||
user = config.get("User", "")
|
||||
candidates = self._python_candidates(source, preferred)
|
||||
storage_error = None
|
||||
try:
|
||||
code, output = self._exec_capture(
|
||||
container_id,
|
||||
["/bin/sh", "-c", _WORKER_PYTHON_PROBE_SHELL, "worker-python-probe", _WORKER_PYTHON_PROBE, *candidates],
|
||||
working_dir,
|
||||
user,
|
||||
)
|
||||
if code == 0:
|
||||
for line in reversed(output.splitlines()):
|
||||
if line.startswith("/") and "\x00" not in line:
|
||||
return line.strip()
|
||||
for line in output.splitlines():
|
||||
prefix = "__DEERFLOW_STORAGE_ERROR__="
|
||||
if line.startswith(prefix):
|
||||
storage_error = line[len(prefix):].strip()
|
||||
except DockerError:
|
||||
# Minimal images may not include a shell; try known Python paths directly.
|
||||
pass
|
||||
|
||||
for candidate in candidates:
|
||||
try:
|
||||
code, output = self._exec_capture(
|
||||
container_id,
|
||||
[candidate, "-c", _WORKER_PYTHON_PROBE],
|
||||
working_dir,
|
||||
user,
|
||||
)
|
||||
except DockerError:
|
||||
continue
|
||||
if code == 0:
|
||||
for line in reversed(output.splitlines()):
|
||||
prefix = "__DEERFLOW_WORKER_PYTHON__="
|
||||
if line.startswith(prefix):
|
||||
resolved = line[len(prefix):].strip()
|
||||
if resolved.startswith("/") and "\x00" not in resolved:
|
||||
return resolved
|
||||
for line in output.splitlines():
|
||||
prefix = "__DEERFLOW_STORAGE_ERROR__="
|
||||
if line.startswith(prefix):
|
||||
storage_error = line[len(prefix):].strip()
|
||||
if storage_error:
|
||||
raise DockerError(storage_error)
|
||||
raise DockerError(
|
||||
"未找到与主服务聊天存储配置兼容的 Worker Python(要求 Python 3.12+,"
|
||||
"且能导入 uvicorn、Worker 入口及当前 SQLite/PostgreSQL 适配器);"
|
||||
"已检查主服务常见 Python 路径和运行进程解释器,请确认 Worker 复用了主服务的镜像、"
|
||||
"环境变量、挂载目录和 Python 环境"
|
||||
)
|
||||
|
||||
def configured_storage_backend(self, source: dict, preferred: str, working_dir: str) -> str:
|
||||
"""Read the live config so SQLite→PostgreSQL changes invalidate probes."""
|
||||
container_id = source.get("Id")
|
||||
if not container_id:
|
||||
raise DockerError("无法读取主后端容器 ID,不能检查聊天存储配置")
|
||||
user = source.get("Config", {}).get("User", "")
|
||||
for candidate in self._python_candidates(source, preferred):
|
||||
try:
|
||||
code, output = self._exec_capture(
|
||||
container_id,
|
||||
[candidate, "-c", _WORKER_STORAGE_PROBE],
|
||||
working_dir,
|
||||
user,
|
||||
)
|
||||
except DockerError:
|
||||
continue
|
||||
if code == 0:
|
||||
for line in reversed(output.splitlines()):
|
||||
prefix = "__DEERFLOW_STORAGE_BACKEND__="
|
||||
if line.startswith(prefix):
|
||||
backend = line[len(prefix):].strip()
|
||||
if backend in {"postgres", "sqlite", "memory"}:
|
||||
return backend
|
||||
raise DockerError(
|
||||
"无法读取主服务当前聊天存储配置;请确认 Python 路径、工作目录和主服务配置文件正确"
|
||||
)
|
||||
|
||||
def snapshot_worker_runtime(self, source: dict, instance: str) -> str:
|
||||
"""Snapshot the running main container's writable layer for new workers.
|
||||
|
||||
Docker create from Config.Image alone loses packages installed into the
|
||||
running container after it was created. A local commit carries those
|
||||
offline-installed packages; bind/volume mounts are still cloned as-is.
|
||||
"""
|
||||
container_id = source.get("Id")
|
||||
if not container_id:
|
||||
raise DockerError("无法读取主后端容器 ID,不能复用其 Python 运行环境")
|
||||
# A freshly started backend often has only a tiny writable overlay
|
||||
# (logs/runtime markers). Committing it can take minutes on an offline
|
||||
# Docker host even though the immutable image already has the runtime.
|
||||
# Keep committing larger overlays that may include offline-installed
|
||||
# Python packages.
|
||||
base_image = source.get("Image")
|
||||
writable_layer_size = source.get("SizeRw")
|
||||
if (
|
||||
isinstance(base_image, str)
|
||||
and base_image
|
||||
and isinstance(writable_layer_size, int)
|
||||
and writable_layer_size <= 128 * 1024
|
||||
):
|
||||
return base_image
|
||||
repo = f"deerflow-worker-runtime-{instance}"
|
||||
tag = container_id[:12]
|
||||
result = self.request(
|
||||
"POST",
|
||||
f"/commit?container={quote(container_id, safe='')}&repo={quote(repo, safe='')}&tag={quote(tag, safe='')}&pause=false",
|
||||
)
|
||||
image_id = result.get("Id") if isinstance(result, dict) else None
|
||||
if not isinstance(image_id, str) or not image_id.startswith("sha256:"):
|
||||
raise DockerError("无法从主容器生成 Worker 运行镜像;请确认 Docker 账号具有容器提交权限")
|
||||
return image_id
|
||||
|
||||
def host_snapshot(self, container: str):
|
||||
"""Only our constant read-only collector may be executed by diagnostics."""
|
||||
if self.settings.transport == "socket":
|
||||
from .host_metrics import read_snapshot
|
||||
return read_snapshot()
|
||||
command = [self.settings.python_executable, "-m", "app.gateway.worker_pool.host_metrics"]
|
||||
created = self.request("POST", f"/containers/{quote(container, safe='')}/exec", {
|
||||
"AttachStdout": True, "AttachStderr": True, "Cmd": command,
|
||||
"WorkingDir": self.settings.working_dir,
|
||||
})
|
||||
data = self.request("POST", f"/exec/{created['Id']}/start", {"Detach": False, "Tty": False}, raw=True, max_bytes=1024 * 1024, json_stream=True)
|
||||
out = decode_docker_output(data)
|
||||
try:
|
||||
return json.loads(out)
|
||||
except ValueError:
|
||||
raise DockerError("宿主机采集器未运行成功,请检查主容器代码和只读 /proc、/sys 挂载") from None
|
||||
|
||||
|
||||
def decode_docker_output(data: bytes) -> str:
|
||||
"""Docker attaches stdout/stderr in 8-byte length-prefixed frames."""
|
||||
import struct
|
||||
if len(data) < 8 or data[0] not in (0, 1, 2) or data[1:4] != b"\x00\x00\x00":
|
||||
return data.decode("utf-8", errors="replace")
|
||||
parts = []
|
||||
offset = 0
|
||||
while offset + 8 <= len(data):
|
||||
size = struct.unpack(">I", data[offset + 4:offset + 8])[0]
|
||||
parts.append(data[offset + 8:offset + 8 + size])
|
||||
offset += 8 + size
|
||||
return b"".join(parts).decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def worker_spec(
|
||||
source: dict,
|
||||
settings: PoolSettings,
|
||||
instance: str,
|
||||
token: str,
|
||||
control_home: str,
|
||||
*,
|
||||
runtime_image: str | None = None,
|
||||
require_shared_archive: bool = False,
|
||||
) -> dict:
|
||||
"""Clone an explicit runtime template, without ports, privileges or Docker access."""
|
||||
if settings.network not in source["NetworkSettings"]["Networks"]:
|
||||
raise DockerError("源容器必须连接所选 Docker 网络")
|
||||
if settings.network in {"host", "none", "bridge"}:
|
||||
raise DockerError("请选择用户自定义 Docker 网络")
|
||||
mounts = []
|
||||
for mount in source.get("Mounts", []):
|
||||
target = mount["Destination"].rstrip("/")
|
||||
if control_home == target:
|
||||
continue # dedicated controller volume is never passed to workers
|
||||
if "docker.sock" in target or target in {"/", "/var/run", "/run"}:
|
||||
continue
|
||||
if target in {"/host/proc", "/host/sys"}:
|
||||
continue # host diagnostics belong to the controller only
|
||||
if control_home.startswith(target + "/"):
|
||||
raise DockerError("控制器密钥目录不能与 Worker 共享,请为 DEER_FLOW_WORKER_CONTROL_HOME 配置独立持久卷")
|
||||
if mount["Type"] not in {"bind", "volume"}:
|
||||
continue
|
||||
mounts.append({"Type": mount["Type"], "Source": mount.get("Name") if mount["Type"] == "volume" else mount["Source"], "Target": mount["Destination"], "ReadOnly": not mount.get("RW", True)})
|
||||
env = dict(item.split("=", 1) for item in source["Config"].get("Env", []) if "=" in item)
|
||||
if not env.get("AUTH_JWT_SECRET"):
|
||||
raise DockerError("请在源容器环境变量中设置固定 AUTH_JWT_SECRET,使 Worker 能验证现有登录会话")
|
||||
if not mounts:
|
||||
raise DockerError("源容器没有共享持久化挂载,无法保证 Worker 访问技能、配置和上传文件")
|
||||
if require_shared_archive:
|
||||
if env.get("DEER_FLOW_CHAT_ARCHIVE_DIR"):
|
||||
archive_path = posixpath.normpath(env["DEER_FLOW_CHAT_ARCHIVE_DIR"])
|
||||
elif env.get("DEER_FLOW_HOME"):
|
||||
archive_path = posixpath.join(env["DEER_FLOW_HOME"], "data", "chat-archives")
|
||||
else:
|
||||
project_root = env.get("DEER_FLOW_PROJECT_ROOT") or source["Config"].get("WorkingDir") or settings.working_dir
|
||||
archive_path = posixpath.join(project_root, ".deer-flow", "data", "chat-archives")
|
||||
if not posixpath.isabs(archive_path):
|
||||
workdir = source["Config"].get("WorkingDir") or settings.working_dir
|
||||
archive_path = posixpath.join(workdir, archive_path)
|
||||
archive_path = posixpath.normpath(archive_path)
|
||||
shared = any(
|
||||
mount.get("ReadOnly") is False
|
||||
and (archive_path == mount["Target"].rstrip("/")
|
||||
or archive_path.startswith(mount["Target"].rstrip("/") + "/"))
|
||||
for mount in mounts
|
||||
)
|
||||
if not shared:
|
||||
raise DockerError(
|
||||
f"PostgreSQL 聊天归档目录 {archive_path} 必须位于主容器与 Worker 共享的可写持久卷中;"
|
||||
"请将 DEER_FLOW_CHAT_ARCHIVE_DIR 指向共享卷目录或挂载 DEER_FLOW_HOME/data"
|
||||
)
|
||||
for key in list(env):
|
||||
if key.startswith("DEER_FLOW_WORKER_") or key in {"DEER_FLOW_REBUILD_WORKFLOW_SCHEMA", "WEB_CONCURRENCY", "UVICORN_WORKERS"}:
|
||||
env.pop(key)
|
||||
env.update(
|
||||
DEER_FLOW_WORKER_DB_POOL_SIZE=str(settings.database_pool_size),
|
||||
DEER_FLOW_PG_POOL_MIN="1",
|
||||
DEER_FLOW_PG_POOL_MAX=str(settings.postgres_pool_size),
|
||||
DEER_FLOW_WORKER_ROLE="online",
|
||||
DEER_FLOW_WORKER_TOKEN=token,
|
||||
DEER_FLOW_WORKER_CONCURRENCY=str(settings.concurrency),
|
||||
DEER_FLOW_WORKER_QUEUE_LIMIT=str(settings.queue_limit),
|
||||
DEER_FLOW_WORKER_QUEUE_TIMEOUT=str(settings.queue_timeout),
|
||||
SCHEDULED_TASKS_ENABLED="false",
|
||||
TASK_ANALYSIS_ENABLED="false",
|
||||
)
|
||||
return {
|
||||
"Image": runtime_image or source["Image"],
|
||||
"WorkingDir": settings.working_dir,
|
||||
"User": source["Config"].get("User", ""),
|
||||
"Entrypoint": [settings.python_executable, "-m", "uvicorn"],
|
||||
"Cmd": ["app.gateway.worker_pool.worker:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1"],
|
||||
"Env": [f"{key}={value}" for key, value in env.items()],
|
||||
"Labels": {"deerflow.worker-pool": instance, "deerflow.worker-role": "online"},
|
||||
"NetworkingConfig": {"EndpointsConfig": {name: {} for name in source["NetworkSettings"]["Networks"] if name not in {"host", "none", "bridge"}}},
|
||||
"Healthcheck": {
|
||||
"Test": ["CMD", settings.python_executable, "-c", "import os,urllib.request; urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:8001/internal/worker-status',headers={'x-deerflow-worker-token':os.environ['DEER_FLOW_WORKER_TOKEN']}),timeout=4).read()"],
|
||||
"Interval": 20_000_000_000, "Timeout": 5_000_000_000, "Retries": 6, "StartPeriod": 90_000_000_000,
|
||||
},
|
||||
"HostConfig": {
|
||||
"Mounts": mounts,
|
||||
"NetworkMode": settings.network,
|
||||
"Memory": settings.memory_mb * 1024 * 1024,
|
||||
"NanoCpus": int(settings.cpu * 1_000_000_000),
|
||||
"PidsLimit": 512,
|
||||
"RestartPolicy": {"Name": "unless-stopped"},
|
||||
"SecurityOpt": ["no-new-privileges:true"],
|
||||
"ExtraHosts": source.get("HostConfig", {}).get("ExtraHosts") or [],
|
||||
"Dns": source.get("HostConfig", {}).get("Dns") or [],
|
||||
"Init": True,
|
||||
"LogConfig": {"Type": "json-file", "Config": {"max-size": "10m", "max-file": "3"}},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Small read-only host snapshot; requires /proc and /sys mounted by the admin."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def read_snapshot(proc=Path("/host/proc"), sys=Path("/host/sys")):
|
||||
if not (proc / "stat").is_file():
|
||||
return {"available": False, "message": "请为主容器挂载 /proc:/host/proc:ro 和 /sys:/host/sys:ro;Worker 不需要挂载。"}
|
||||
cpu = next(line for line in (proc / "stat").read_text().splitlines() if line.startswith("cpu "))
|
||||
fields = [int(v) for v in cpu.split()[1:9]] # guest times are already included in user/nice
|
||||
mem = {}
|
||||
for line in (proc / "meminfo").read_text().splitlines():
|
||||
k, value = line.split(":", 1)
|
||||
mem[k] = int(value.split()[0]) * 1024
|
||||
disks = {}
|
||||
for line in (proc / "diskstats").read_text().splitlines():
|
||||
v = line.split()
|
||||
if len(v) < 14 or re.match(r"^(loop|ram|fd|sr)", v[2]):
|
||||
continue
|
||||
# Whole devices only: including partitions would double-count disk IO.
|
||||
if (sys / "class/block" / v[2] / "partition").exists():
|
||||
continue
|
||||
disks[v[2]] = {"read_bytes": int(v[5]) * 512, "write_bytes": int(v[9]) * 512,
|
||||
"io_ms": int(v[12]), "inflight": int(v[11])}
|
||||
interfaces = {}
|
||||
network_error = None
|
||||
try:
|
||||
# /proc/1 is the host PID namespace; /proc/net would resolve to this container.
|
||||
for line in (proc / "1/net/dev").read_text().splitlines()[2:]:
|
||||
name, values = line.split(":", 1)
|
||||
name = name.strip()
|
||||
if name == "lo" or name.startswith(("veth", "docker", "br-", "virbr", "tun", "tap")):
|
||||
continue
|
||||
v = values.split()
|
||||
speed = None
|
||||
try:
|
||||
speed = int((sys / "class/net" / name / "speed").read_text())
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
interfaces[name] = {"rx_bytes": int(v[0]), "tx_bytes": int(v[8]),
|
||||
"rx_errors": int(v[2]), "tx_errors": int(v[10]), "link_mbps": speed if speed and speed > 0 else None}
|
||||
except OSError:
|
||||
network_error = "无法读取宿主机网络命名空间,检查 /host/proc 挂载及读取权限"
|
||||
pressure = {}
|
||||
for name in ("cpu", "memory", "io"):
|
||||
try:
|
||||
pressure[name] = (proc / "pressure" / name).read_text().strip()[:400]
|
||||
except OSError:
|
||||
pressure[name] = None
|
||||
processes = {}
|
||||
page_size = getattr(os, "sysconf", lambda _: 4096)("SC_PAGE_SIZE")
|
||||
for item in proc.iterdir():
|
||||
if not item.name.isdigit():
|
||||
continue
|
||||
if len(processes) >= 5000:
|
||||
break
|
||||
try:
|
||||
stat = (item / "stat").read_text()
|
||||
end = stat.rfind(")")
|
||||
process_fields = stat[end + 2:].split()
|
||||
processes[item.name] = {"name": stat[stat.find("(") + 1:end][:80], "ticks": int(process_fields[11]) + int(process_fields[12]),
|
||||
"started": int(process_fields[19]), "rss_bytes": max(0, int(process_fields[21])) * page_size}
|
||||
except (OSError, ValueError, IndexError):
|
||||
continue
|
||||
return {"available": True, "at": time.time(), "cpu_total": sum(fields), "cpu_idle": fields[3] + fields[4],
|
||||
"cpu_iowait": fields[4], "cpu_count": sum(line.startswith("cpu") and line[3:4].isdigit() for line in (proc / "stat").read_text().splitlines()),
|
||||
"memory_total": mem.get("MemTotal", 0), "memory_available": mem.get("MemAvailable", mem.get("MemFree", 0)),
|
||||
"swap_total": mem.get("SwapTotal", 0), "swap_free": mem.get("SwapFree", 0),
|
||||
"load": [float(v) for v in (proc / "loadavg").read_text().split()[:3]],
|
||||
"disks": disks, "interfaces": interfaces, "network_error": network_error, "pressure": pressure,
|
||||
"process_counters": processes, "clock_ticks": getattr(os, "sysconf", lambda _: 100)("SC_CLK_TCK")}
|
||||
|
||||
|
||||
def rates(current, previous):
|
||||
result = dict(current)
|
||||
if not current.get("available"):
|
||||
return result
|
||||
result.update(cpu_percent=None, iowait_percent=None)
|
||||
elapsed = current["at"] - previous["at"] if previous and previous.get("available") else 0
|
||||
if elapsed > 0:
|
||||
delta = current["cpu_total"] - previous["cpu_total"]
|
||||
if delta > 0:
|
||||
result["cpu_percent"] = round(100 * (1 - (current["cpu_idle"] - previous["cpu_idle"]) / delta), 2)
|
||||
result["iowait_percent"] = round(100 * (current["cpu_iowait"] - previous["cpu_iowait"]) / delta, 2)
|
||||
top = []
|
||||
for pid, item in current.get("process_counters", {}).items():
|
||||
old = previous.get("process_counters", {}).get(pid) if elapsed > 0 else None
|
||||
if old and old["started"] == item["started"]:
|
||||
percent = max(0, (item["ticks"] - old["ticks"]) / (current.get("clock_ticks", 100) * elapsed) * 100)
|
||||
top.append({"pid": int(pid), "name": item["name"], "cpu_percent": round(percent, 2), "rss_bytes": item["rss_bytes"]})
|
||||
result.pop("process_counters", None)
|
||||
result["top_processes"] = sorted(top, key=lambda p: p["cpu_percent"], reverse=True)[:20]
|
||||
for name, item in result["disks"].items():
|
||||
old = previous.get("disks", {}).get(name) if elapsed > 0 else None
|
||||
item.update(read_bps=max(0, (item["read_bytes"] - old["read_bytes"]) / elapsed) if old else None,
|
||||
write_bps=max(0, (item["write_bytes"] - old["write_bytes"]) / elapsed) if old else None,
|
||||
busy_percent=min(100, max(0, (item["io_ms"] - old["io_ms"]) / (elapsed * 10))) if old else None)
|
||||
for name, item in result["interfaces"].items():
|
||||
old = previous.get("interfaces", {}).get(name) if elapsed > 0 else None
|
||||
item.update(rx_bps=max(0, (item["rx_bytes"] - old["rx_bytes"]) / elapsed) if old else None,
|
||||
tx_bps=max(0, (item["tx_bytes"] - old["tx_bytes"]) / elapsed) if old else None)
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(json.dumps(read_snapshot()))
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Create the opt-in Worker controller without restricting normal startup."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _control_directory() -> Path:
|
||||
return Path(os.getenv("DEER_FLOW_WORKER_CONTROL_HOME", ".worker-pool")).resolve()
|
||||
|
||||
|
||||
def _persisted_feature_enabled(directory: Path) -> bool:
|
||||
def read(path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
state = read(directory / "settings.json")
|
||||
diagnostics = read(directory / "diagnostics-settings.json")
|
||||
return bool(state.get("settings", {}).get("enabled") or diagnostics.get("enabled"))
|
||||
|
||||
|
||||
def ensure_controller(app):
|
||||
pool = getattr(app.state, "worker_pool", None)
|
||||
if pool is not None:
|
||||
return pool
|
||||
from .controller import Controller
|
||||
from .diagnostics import Diagnostics
|
||||
from .settings import SettingsStore
|
||||
|
||||
directory = getattr(app.state, "worker_pool_directory", _control_directory())
|
||||
pool = Controller(SettingsStore(directory))
|
||||
pool.diagnostics = Diagnostics(pool, app)
|
||||
app.state.worker_pool = pool
|
||||
return pool
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def managed_controller(app):
|
||||
directory = _control_directory()
|
||||
app.state.worker_pool_directory = directory
|
||||
app.state.worker_pool = None
|
||||
pool = ensure_controller(app) if _persisted_feature_enabled(directory) else None
|
||||
try:
|
||||
yield pool
|
||||
finally:
|
||||
pool = getattr(app.state, "worker_pool", None)
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""A pinned online worker can identify its own orphaned in-flight records."""
|
||||
|
||||
import inspect
|
||||
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
from deerflow.runtime.runs.schemas import RunStatus
|
||||
|
||||
|
||||
class WorkerRunManager(RunManager):
|
||||
def has_active_thread(self, thread_id: str) -> bool:
|
||||
return any(
|
||||
record.thread_id == thread_id
|
||||
and record.status in {RunStatus.pending, RunStatus.running}
|
||||
and (record.task is None or not record.task.done())
|
||||
for record in self._runs.values()
|
||||
)
|
||||
|
||||
async def get_persisted(self, run_id: str, *, authorized_thread_id: str | None = None):
|
||||
if "authorized_thread_id" in inspect.signature(RunManager.get_persisted).parameters:
|
||||
record = await super().get_persisted(run_id, authorized_thread_id=authorized_thread_id)
|
||||
else:
|
||||
record = await super().get_persisted(run_id)
|
||||
if record is not None and authorized_thread_id and record.thread_id != authorized_thread_id:
|
||||
return None
|
||||
if record is not None and self.get(run_id) is None and record.status in {RunStatus.pending, RunStatus.running}:
|
||||
# Routing never migrates a live process. Missing live state means a
|
||||
# previous process ended; replaying tools here would be unsafe.
|
||||
record.status = RunStatus.error
|
||||
record.error = "执行节点已重启,本轮已中断;已保存的聊天记录保留。请核对外部操作结果后再继续。"
|
||||
await self._store.update_status(run_id, record.status.value, error=record.error)
|
||||
return record
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Match the runtime's legacy-first persistence configuration precedence."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def postgres_connection_string(config) -> str | None:
|
||||
if not shared_postgres_configured(config):
|
||||
return None
|
||||
checkpoint = getattr(config, "checkpointer", None)
|
||||
if checkpoint is not None:
|
||||
return getattr(checkpoint, "connection_string", None)
|
||||
return getattr(config.database, "postgres_url", None)
|
||||
|
||||
|
||||
def shared_postgres_configured(config) -> bool:
|
||||
database = getattr(config, "database", None)
|
||||
if getattr(database, "backend", None) not in {"postgres", "mysql"}:
|
||||
return False
|
||||
checkpoint = getattr(config, "checkpointer", None)
|
||||
if checkpoint is None:
|
||||
return database.backend == "postgres"
|
||||
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
|
||||
enabled = raw.strip().lower() not in {"0", "false", "no", "off"} if raw else getattr(checkpoint, "use_postgres", True)
|
||||
return checkpoint.type == "postgres" and enabled
|
||||
|
||||
|
||||
def sqlite_checkpoint_path(config) -> str | None:
|
||||
from deerflow.runtime.checkpointer.async_provider import _sqlite_conn_str_for_postgres_fallback, _use_postgres_enabled
|
||||
from deerflow.runtime.store._sqlite_utils import resolve_sqlite_conn_str
|
||||
|
||||
checkpoint = getattr(config, "checkpointer", None)
|
||||
if checkpoint is None:
|
||||
return None
|
||||
if checkpoint.type == "sqlite":
|
||||
raw = getattr(checkpoint, "connection_string", None) or "store.db"
|
||||
elif checkpoint.type == "postgres" and not _use_postgres_enabled(checkpoint):
|
||||
raw = _sqlite_conn_str_for_postgres_fallback(checkpoint)
|
||||
else:
|
||||
return None
|
||||
return resolve_sqlite_conn_str(raw)
|
||||
|
||||
|
||||
def shared_storage_configured(config) -> bool:
|
||||
if shared_postgres_configured(config):
|
||||
return True
|
||||
if getattr(getattr(config, "database", None), "backend", None) not in {"mysql", "postgres"}:
|
||||
return False
|
||||
path = sqlite_checkpoint_path(config)
|
||||
return bool(path and path != ":memory:" and not path.startswith("file:"))
|
||||
|
||||
|
||||
def validate_sqlite_mount(config, mountinfo: str | None = None) -> None:
|
||||
"""SQLite databases and WAL sidecars must live on one shared local directory."""
|
||||
path = sqlite_checkpoint_path(config)
|
||||
if path is None:
|
||||
return
|
||||
database = Path(path).resolve()
|
||||
content = mountinfo if mountinfo is not None else Path("/proc/self/mountinfo").read_text()
|
||||
for line in content.splitlines():
|
||||
left, right = line.split(" - ", 1)
|
||||
mount = Path(left.split()[4].replace("\\040", " "))
|
||||
# A single-file bind does not share the -wal/-shm files.
|
||||
if mount != Path("/") and mount in database.parents:
|
||||
if right.split()[0] in {"nfs", "nfs4", "cifs", "smb3", "9p"}:
|
||||
raise RuntimeError("SQLite Worker requires a shared local directory, not a network filesystem")
|
||||
return
|
||||
raise RuntimeError("SQLite Worker requires the database directory to be persistently mounted in every container")
|
||||
@@ -0,0 +1,153 @@
|
||||
"""ASGI streaming proxy. Preserve authentication, disconnects and SSE headers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from .controller import PoolUnavailable
|
||||
|
||||
THREAD_PATH = re.compile(r"^/api/threads/([^/]+)(?:/.*)?$")
|
||||
START_PATH = re.compile(r"^/api/threads/[^/]+/runs(?:/(?:stream|wait))?$")
|
||||
HOP_HEADERS = {b"host", b"connection", b"keep-alive", b"transfer-encoding", b"upgrade", b"proxy-authenticate", b"proxy-authorization", b"te", b"trailer", b"content-length", b"x-deerflow-worker-token"}
|
||||
|
||||
|
||||
def _main_has_persistent_chat_storage(app) -> bool:
|
||||
checkpointer = getattr(app.state, "checkpointer", None)
|
||||
store = getattr(app.state, "store", None)
|
||||
if checkpointer is None or store is None:
|
||||
return False
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
|
||||
return not isinstance(checkpointer, InMemorySaver) and not isinstance(store, InMemoryStore)
|
||||
|
||||
|
||||
async def send_upstream(client, method, url, headers, content):
|
||||
# A keep-alive socket may close while a completed run is being queried.
|
||||
# Retry only body-free reads, before downstream headers have been sent.
|
||||
attempts = 2 if method in {"GET", "HEAD"} else 1
|
||||
for attempt in range(attempts):
|
||||
upstream = client.build_request(method, url, headers=headers, content=None if method in {"GET", "HEAD"} else content)
|
||||
try:
|
||||
return await client.send(upstream, stream=True)
|
||||
except httpx.TransportError:
|
||||
if attempt + 1 == attempts:
|
||||
raise
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
|
||||
class WorkerProxy:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
pool = getattr(scope["app"].state, "worker_pool", None)
|
||||
match = THREAD_PATH.fullmatch(scope["path"])
|
||||
if pool is None or match is None or match[1] in {"search", "count", "reference-debug-log"}:
|
||||
return await self.app(scope, receive, send)
|
||||
if pool.db is None and not pool.store.load().enabled:
|
||||
return await self.app(scope, receive, send)
|
||||
row = pool._ensure_db().execute("SELECT node FROM affinity WHERE thread=?", (match[1],)).fetchone()
|
||||
creating = scope["method"] == "POST" and bool(START_PATH.fullmatch(scope["path"]))
|
||||
if row is None and (not creating or not pool.store.load().enabled):
|
||||
return await self.app(scope, receive, send)
|
||||
# Authenticate before allocating durable routing state. The worker
|
||||
# still performs the existing owner/permission checks for every route.
|
||||
request = Request(scope, receive)
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.deps import get_current_user_from_request, get_thread_store
|
||||
|
||||
try:
|
||||
access_started = time.perf_counter()
|
||||
user = getattr(request.state, "user", None) or await get_current_user_from_request(request)
|
||||
allowed = await get_thread_store(request).check_access(match[1], str(user.id), require_existing=True)
|
||||
if not allowed:
|
||||
return await JSONResponse({"detail": "Thread not found"}, 404)(scope, receive, send)
|
||||
if row is None:
|
||||
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
|
||||
|
||||
record = await get_thread_store(request).get(match[1], user_id=None)
|
||||
if record and is_excluded_system_thread(record.get("metadata")):
|
||||
return await self.app(scope, receive, send)
|
||||
manager = getattr(scope["app"].state, "run_manager", None)
|
||||
if creating and pool.local_threads.get(match[1], 0):
|
||||
return await JSONResponse({"detail": "会话仍在主服务执行,请完成当前轮后再提交"}, 409)(scope, receive, send)
|
||||
if creating and manager is not None and any(
|
||||
record.thread_id == match[1]
|
||||
and getattr(record.status, "value", record.status) in {"pending", "running"}
|
||||
and (record.task is None or not record.task.done())
|
||||
for record in manager._runs.values()
|
||||
):
|
||||
return await JSONResponse({"detail": "会话仍在执行,请完成当前轮后再提交"}, 409)(scope, receive, send)
|
||||
node = await pool.reserve(match[1], create=creating)
|
||||
scope.setdefault("state", {})["worker_access_ms"] = round((time.perf_counter() - access_started) * 1000, 2)
|
||||
except HTTPException as exc:
|
||||
return await JSONResponse({"detail": exc.detail}, exc.status_code)(scope, receive, send)
|
||||
except PoolUnavailable as exc:
|
||||
if creating and pool.local_threads.get(match[1], 0):
|
||||
return await JSONResponse({"detail": str(exc)}, 409)(scope, receive, send)
|
||||
return await JSONResponse({"detail": str(exc)}, 503, headers={"Retry-After": "3"})(scope, receive, send)
|
||||
if node is None:
|
||||
return await self.app(scope, receive, send)
|
||||
if node.get("_fallback_to_main"):
|
||||
fallback_worker = node.get("_worker")
|
||||
if creating and fallback_worker is not None:
|
||||
try:
|
||||
active_response = await pool.http.get(
|
||||
fallback_worker["url"] + "/internal/worker-thread-active/" + quote(match[1], safe=""),
|
||||
headers={"x-deerflow-worker-token": pool.store.raw()["token"]},
|
||||
timeout=2,
|
||||
)
|
||||
active_response.raise_for_status()
|
||||
if active_response.json().get("active"):
|
||||
return await JSONResponse({"detail": "该会话正在 Worker 执行,请等待当前回答完成"}, 409)(scope, receive, send)
|
||||
except httpx.HTTPError:
|
||||
return await JSONResponse({"detail": "无法确认该会话是否正在 Worker 执行,请稍后重试"}, 503, headers={"Retry-After": "2"})(scope, receive, send)
|
||||
if not _main_has_persistent_chat_storage(scope["app"]):
|
||||
return await JSONResponse(
|
||||
{"detail": "主服务的持久化聊天存储当前不可用,无法安全接管 Worker 请求"},
|
||||
503,
|
||||
headers={"Retry-After": "3"},
|
||||
)(scope, receive, send)
|
||||
scope.setdefault("state", {})["worker_pool_fallback"] = True
|
||||
scope["state"]["worker_pool_fallback_reason"] = node.get("_fallback_reason")
|
||||
return await self.app(scope, receive, send)
|
||||
response = None
|
||||
scope.setdefault("state", {})["worker_node"] = node["name"]
|
||||
try:
|
||||
headers = [(k, v) for k, v in scope["headers"] if k.lower() not in HOP_HEADERS or k.lower() == b"host"]
|
||||
headers.append((b"x-deerflow-worker-token", pool.store.raw()["token"].encode()))
|
||||
url = node["url"] + scope["path"]
|
||||
if scope.get("query_string"):
|
||||
url += "?" + scope["query_string"].decode("ascii")
|
||||
upstream_started = time.perf_counter()
|
||||
response = await send_upstream(pool.http, scope["method"], url, headers, request.stream())
|
||||
scope["state"]["worker_upstream_headers_ms"] = round((time.perf_counter() - upstream_started) * 1000, 2)
|
||||
|
||||
async def chunks():
|
||||
async for chunk in response.aiter_raw():
|
||||
yield chunk
|
||||
|
||||
downstream = StreamingResponse(chunks(), status_code=response.status_code)
|
||||
downstream.raw_headers = [(k, v) for k, v in response.headers.raw if k.lower() not in HOP_HEADERS]
|
||||
await downstream(scope, receive, send)
|
||||
except httpx.HTTPError as exc:
|
||||
logging.getLogger(__name__).warning("Worker transport failed method=%s node=%s error=%s", scope["method"], node["name"], type(exc).__name__)
|
||||
if response is None:
|
||||
await JSONResponse({"detail": "执行节点连接失败。任务可能已开始,请查看会话状态后再决定是否重试。"}, 503)(scope, receive, send)
|
||||
# Once SSE headers are sent, close the stream so the SDK reconnects.
|
||||
finally:
|
||||
if response is not None:
|
||||
await response.aclose()
|
||||
pool.release(node["name"])
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Observe shared skill edits before accepting a new model run."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
|
||||
class SharedSkillRefresh:
|
||||
def __init__(self):
|
||||
self.lock = asyncio.Lock()
|
||||
self.extensions = None
|
||||
self.skills = None
|
||||
self.checked_at = 0.0
|
||||
|
||||
@staticmethod
|
||||
def extension_stamp():
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
|
||||
path = ExtensionsConfig.resolve_config_path()
|
||||
if path is None:
|
||||
return None
|
||||
stat = path.stat()
|
||||
return (str(path), stat.st_mtime_ns, stat.st_size)
|
||||
|
||||
@staticmethod
|
||||
def skill_stamp():
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
root = Path(get_or_new_skill_storage().get_skills_root_path())
|
||||
stamps = []
|
||||
for category in ("public", "custom"):
|
||||
for path in (root / category).rglob("*"):
|
||||
if path.suffix in {".md", ".py"} and path.is_file():
|
||||
stat = path.stat()
|
||||
stamps.append((str(path), stat.st_mtime_ns, stat.st_size))
|
||||
return tuple(sorted(stamps))
|
||||
|
||||
async def check(self):
|
||||
async with self.lock:
|
||||
extension = await asyncio.to_thread(self.extension_stamp)
|
||||
skills = self.skills
|
||||
if time.monotonic() - self.checked_at >= 5:
|
||||
skills = await asyncio.to_thread(self.skill_stamp)
|
||||
self.checked_at = time.monotonic()
|
||||
if extension != self.extensions or skills != self.skills:
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.config.extensions_config import reload_extensions_config
|
||||
|
||||
await asyncio.to_thread(reload_extensions_config)
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
self.extensions, self.skills = extension, skills
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Fingerprint this worker's execution adapter at process startup."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def code_revision():
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(Path(__file__).parent.glob("*.py")):
|
||||
digest.update(path.name.encode())
|
||||
digest.update(path.read_bytes())
|
||||
return digest.hexdigest()[:12]
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Strict admin-only management; anonymous mode never grants host access."""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.gateway.deps import get_current_user_from_request
|
||||
|
||||
from .controller import PoolUnavailable
|
||||
from .docker import DockerError
|
||||
from .preflight import shared_storage_configured
|
||||
from .settings import PoolSettings
|
||||
|
||||
router = APIRouter(prefix="/api/system-settings/worker-pool", tags=["worker-pool"])
|
||||
|
||||
|
||||
class DiagnosticsSwitch(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
@router.get("/diagnostics")
|
||||
async def diagnostics_status(request: Request):
|
||||
return (await controller(request)).diagnostics.status()
|
||||
|
||||
|
||||
@router.put("/diagnostics")
|
||||
async def diagnostics_switch(body: DiagnosticsSwitch, request: Request):
|
||||
pool = await controller(request)
|
||||
await pool.diagnostics.set_enabled(body.enabled)
|
||||
return pool.diagnostics.status()
|
||||
|
||||
|
||||
@router.get("/logs")
|
||||
async def container_logs(request: Request, node: str = Query(min_length=1, max_length=128), tail: int = Query(default=200, ge=1, le=1000), minutes: int = Query(default=60, ge=1, le=1440), query: str = Query(default="", max_length=100)):
|
||||
pool = await controller(request)
|
||||
try:
|
||||
return await pool.diagnostics.logs(node, tail, minutes, query)
|
||||
except DockerError as exc:
|
||||
raise HTTPException(400, str(exc)) from None
|
||||
|
||||
|
||||
@router.get("/log-archives")
|
||||
async def log_archives(request: Request):
|
||||
return (await controller(request)).diagnostics.archives()
|
||||
|
||||
|
||||
async def controller(request: Request):
|
||||
user = await get_current_user_from_request(request)
|
||||
if user is None or getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(403, "仅管理员可以管理执行节点")
|
||||
from .lifecycle import ensure_controller
|
||||
|
||||
return ensure_controller(request.app)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def status(request: Request):
|
||||
return (await controller(request)).status()
|
||||
|
||||
|
||||
@router.put("")
|
||||
async def save(body: PoolSettings, request: Request):
|
||||
pool = await controller(request)
|
||||
async with pool.lock:
|
||||
current = pool.store.load()
|
||||
immutable = ("transport", "host", "port", "username", "source_container", "network", "concurrency", "cpu", "memory_mb", "queue_limit", "queue_timeout", "python_executable", "working_dir", "database_pool_size", "postgres_pool_size")
|
||||
controller_active = pool.task is not None and not pool.task.done()
|
||||
if (pool.nodes or (controller_active and not pool.discovery_complete)) and any(getattr(current, key) != getattr(body, key) for key in immutable):
|
||||
raise HTTPException(409, "请先将节点数设为 0,等待所有节点退出后再修改连接或单节点配置")
|
||||
config = getattr(request.app.state, "config", None)
|
||||
if body.enabled:
|
||||
if not shared_storage_configured(config):
|
||||
raise HTTPException(409, "启用前必须配置持久化 PostgreSQL 或 SQLite 聊天存储及共享业务库")
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
if isinstance(getattr(request.app.state, "checkpointer", None), InMemorySaver):
|
||||
raise HTTPException(409, "当前检查点已降级到内存,不能启用 Worker")
|
||||
if "AioSandboxProvider" in str(getattr(getattr(config, "sandbox", None), "use", "")):
|
||||
raise HTTPException(409, "当前沙箱需要 Docker 管理权限,在线 Worker 暂不支持;请先配置不依赖 Docker socket 的沙箱")
|
||||
if body.desired * body.cpu > body.total_cpu or body.desired * body.memory_mb > body.total_memory_mb:
|
||||
raise HTTPException(422, "Worker 总资源超过预算")
|
||||
if body.desired * (body.database_pool_size + 2 * body.postgres_pool_size) > body.database_connection_budget:
|
||||
raise HTTPException(422, "Worker 数据库连接总额度超过预算")
|
||||
identity = ("transport", "host", "port", "username", "host_key_sha256")
|
||||
if body.transport == "ssh" and body.password is None and current.password and any(getattr(body, k) != getattr(current, k) for k in identity):
|
||||
raise HTTPException(422, "服务器连接身份已改变,请重新填写密码")
|
||||
pool.store.save(body)
|
||||
try:
|
||||
pool.start()
|
||||
except RuntimeError as exc:
|
||||
pool.store.save(current)
|
||||
raise HTTPException(409, str(exc)) from None
|
||||
return pool.status()
|
||||
|
||||
|
||||
@router.post("/test")
|
||||
async def test_connection(request: Request):
|
||||
pool = await controller(request)
|
||||
try:
|
||||
async with pool.lock:
|
||||
await pool.inspect_template(pool.store.load())
|
||||
except (DockerError, PoolUnavailable) as exc:
|
||||
raise HTTPException(400, str(exc)) from None
|
||||
return {"ok": True, "message": "服务器连接和容器模板检查通过,镜像内 Python 路径已自动校验;节点启动后还会检查聊天存储与运行状态"}
|
||||
|
||||
|
||||
@router.post("/fingerprint")
|
||||
async def fingerprint(body: PoolSettings, request: Request):
|
||||
import asyncio
|
||||
|
||||
from .discovery import host_fingerprint
|
||||
await controller(request)
|
||||
try:
|
||||
value = await asyncio.to_thread(host_fingerprint, body.host, body.port)
|
||||
except DockerError as exc:
|
||||
raise HTTPException(400, str(exc)) from None
|
||||
return {"fingerprint": value}
|
||||
|
||||
|
||||
@router.post("/discover")
|
||||
async def discover_server(body: PoolSettings, request: Request):
|
||||
import asyncio
|
||||
|
||||
from .discovery import discover
|
||||
pool = await controller(request)
|
||||
current = pool.store.load()
|
||||
if body.password is None:
|
||||
identity = ("transport", "host", "port", "username", "host_key_sha256")
|
||||
if all(getattr(body, k) == getattr(current, k) for k in identity):
|
||||
body = body.model_copy(update={"password": current.password})
|
||||
try:
|
||||
return await asyncio.to_thread(discover, body, pool.store.raw()["instance"])
|
||||
except DockerError as exc:
|
||||
raise HTTPException(400, str(exc)) from None
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Runtime checks shared by the controller's interpreter probe and workers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
|
||||
def storage_backend(config) -> str:
|
||||
"""Return the effective LangGraph persistence adapter selected by config."""
|
||||
checkpoint = getattr(config, "checkpointer", None)
|
||||
if checkpoint is not None:
|
||||
if checkpoint.type == "postgres":
|
||||
from deerflow.runtime.checkpointer.async_provider import _use_postgres_enabled
|
||||
|
||||
return "postgres" if _use_postgres_enabled(checkpoint) else "sqlite"
|
||||
if checkpoint.type == "sqlite":
|
||||
return "sqlite"
|
||||
return "memory"
|
||||
|
||||
database = getattr(config, "database", None)
|
||||
backend = getattr(database, "backend", "memory")
|
||||
return backend if backend in {"postgres", "sqlite"} else "memory"
|
||||
|
||||
|
||||
def verify_storage_dependencies(config) -> str:
|
||||
"""Verify the adapters the configured runtime will import, without DB access."""
|
||||
backend = storage_backend(config)
|
||||
modules = {
|
||||
"postgres": (
|
||||
"langgraph.checkpoint.postgres.aio",
|
||||
"langgraph.store.postgres.aio",
|
||||
"psycopg.rows",
|
||||
"psycopg_pool",
|
||||
),
|
||||
"sqlite": (
|
||||
"langgraph.checkpoint.sqlite.aio",
|
||||
"langgraph.store.sqlite.aio",
|
||||
),
|
||||
}.get(backend, ())
|
||||
for module in modules:
|
||||
try:
|
||||
importlib.import_module(module)
|
||||
except ImportError as exc:
|
||||
install = (
|
||||
"langgraph-checkpoint-postgres psycopg[binary] psycopg-pool"
|
||||
if backend == "postgres"
|
||||
else "langgraph-checkpoint-sqlite aiosqlite"
|
||||
)
|
||||
raise RuntimeError(
|
||||
f"Worker Python is missing {module} required by the configured "
|
||||
f"{backend} chat store; install {install} in the main service "
|
||||
"runtime and ensure Workers use that same Python environment"
|
||||
) from exc
|
||||
return backend
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Private controller configuration. Never expose credentials through GET APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class PoolSettings(BaseModel):
|
||||
enabled: bool = False
|
||||
transport: Literal["ssh", "socket"] = "ssh"
|
||||
host: str = ""
|
||||
port: int = Field(default=22, ge=1, le=65535)
|
||||
username: str = ""
|
||||
host_key_sha256: str = ""
|
||||
source_container: str = ""
|
||||
network: str = ""
|
||||
python_executable: str = Field(default="/opt/venv/bin/python", pattern=r"^/[a-zA-Z0-9_./-]+$")
|
||||
working_dir: str = Field(default="/app/backend", pattern=r"^/[a-zA-Z0-9_./-]+$")
|
||||
desired: int = Field(default=0, ge=0, le=32)
|
||||
concurrency: int = Field(default=4, ge=1, le=32)
|
||||
database_pool_size: int = Field(default=4, ge=1, le=32)
|
||||
postgres_pool_size: int = Field(default=4, ge=1, le=32)
|
||||
database_connection_budget: int = Field(default=64, ge=3, le=1024)
|
||||
queue_limit: int = Field(default=128, ge=1, le=1024)
|
||||
queue_timeout: int = Field(default=60, ge=1, le=300)
|
||||
cpu: float = Field(default=2, ge=0.25, le=64)
|
||||
memory_mb: int = Field(default=2048, ge=512, le=131072)
|
||||
total_cpu: float = Field(default=8, ge=0.25, le=512)
|
||||
total_memory_mb: int = Field(default=8192, ge=512, le=1048576)
|
||||
password: str | None = Field(default=None, max_length=4096, repr=False)
|
||||
|
||||
@field_validator("source_container", "network")
|
||||
@classmethod
|
||||
def docker_name(cls, value: str) -> str:
|
||||
if value and not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}", value):
|
||||
raise ValueError("容器名和网络名只能包含字母、数字、点、横线、下划线")
|
||||
return value
|
||||
|
||||
|
||||
class SettingsStore:
|
||||
def __init__(self, directory: Path | None = None):
|
||||
self.directory = directory or Path(os.getenv("DEER_FLOW_WORKER_CONTROL_HOME", ".worker-pool")).resolve()
|
||||
self.directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
key_path = self.directory / "secret.key"
|
||||
try:
|
||||
fd = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
except FileExistsError:
|
||||
pass
|
||||
else:
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
out.write(Fernet.generate_key())
|
||||
self.cipher = Fernet(key_path.read_bytes())
|
||||
self.path = self.directory / "settings.json"
|
||||
if not self.path.exists():
|
||||
self._write({"instance": secrets.token_hex(8), "token": secrets.token_urlsafe(48), "settings": {}})
|
||||
|
||||
def _write(self, data: dict) -> None:
|
||||
temp = self.path.with_suffix(".tmp")
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as out:
|
||||
json.dump(data, out)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
os.replace(temp, self.path)
|
||||
|
||||
def raw(self) -> dict:
|
||||
return json.loads(self.path.read_text(encoding="utf-8"))
|
||||
|
||||
def load(self) -> PoolSettings:
|
||||
data = self.raw()["settings"].copy()
|
||||
encrypted = data.pop("encrypted_password", None)
|
||||
if encrypted:
|
||||
data["password"] = self.cipher.decrypt(encrypted.encode()).decode()
|
||||
return PoolSettings(**data)
|
||||
|
||||
def save(self, settings: PoolSettings) -> None:
|
||||
data = self.raw()
|
||||
values = settings.model_dump(exclude={"password"})
|
||||
password = settings.password if settings.password is not None else self.load().password
|
||||
if password:
|
||||
values["encrypted_password"] = self.cipher.encrypt(password.encode()).decode()
|
||||
data["settings"] = values
|
||||
self._write(data)
|
||||
|
||||
def public(self) -> dict:
|
||||
settings = self.load()
|
||||
return {**settings.model_dump(exclude={"password"}), "password_set": bool(settings.password)}
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Opt-in, bounded HTTP timing. Never consumes/rebuffers a streaming body."""
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
|
||||
from app.gateway.perf_logging import is_verbose_path, should_log_request, verbose_request_logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def route_name(path):
|
||||
path = re.sub(r"(/api/threads/)(?!search(?:/|$)|count(?:/|$))[^/]+", r"\1{thread_id}", path)
|
||||
path = re.sub(r"(/runs/)[^/]+(?=/|$)", r"\1{run_id}", path) if not path.endswith(("/runs/stream", "/runs/wait")) else path
|
||||
return re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27,}", "{id}", path, flags=re.I)[:240]
|
||||
|
||||
|
||||
def percentile(values, percent):
|
||||
values = sorted(v for v in values if v is not None)
|
||||
return round(values[max(0, math.ceil(len(values) * percent / 100) - 1)], 2) if values else None
|
||||
|
||||
|
||||
def _database_pool_snapshot():
|
||||
try:
|
||||
from deerflow.persistence.engine import get_engine
|
||||
|
||||
engine = get_engine()
|
||||
if engine is None:
|
||||
return None
|
||||
pool = engine.sync_engine.pool
|
||||
return {
|
||||
"backend": engine.dialect.name,
|
||||
"checked_out": pool.checkedout(),
|
||||
"size": pool.size(),
|
||||
"overflow": pool.overflow(),
|
||||
"max_overflow": getattr(pool, "_max_overflow", None),
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class RequestRecorder:
|
||||
def __init__(self):
|
||||
self.enabled = False
|
||||
self.completed = deque(maxlen=2000)
|
||||
self.pending = deque(maxlen=4000)
|
||||
self.active = {}
|
||||
|
||||
def finish(self, record):
|
||||
self.completed.append(record)
|
||||
self.pending.append(record)
|
||||
self.active.pop(record["id"], None)
|
||||
|
||||
def summary(self):
|
||||
now = time.time()
|
||||
rows = [r for r in self.completed if r["started"] >= now - 900]
|
||||
grouped = {}
|
||||
for r in rows:
|
||||
# Preserve where the request actually ran. A gateway-only
|
||||
# aggregate hides a saturated Worker behind a healthy main node.
|
||||
key = (r.get("node") or "main", r["method"], r["path"], r["stream"])
|
||||
grouped.setdefault(key, []).append(r)
|
||||
aggregates = []
|
||||
for (node, method, path, stream), records in grouped.items():
|
||||
aggregates.append({"node": node, "method": method, "path": path, "stream": stream, "count": len(records),
|
||||
"errors": sum(r["status"] >= 400 for r in records),
|
||||
"p95_headers_ms": percentile([r["headers_ms"] for r in records], 95),
|
||||
"p95_ms": percentile([r["total_ms"] for r in records], 95),
|
||||
"max_ms": max(r["total_ms"] for r in records)})
|
||||
return {"window_seconds": 900, "count": len(rows), "errors": sum(r["status"] >= 400 for r in rows),
|
||||
"p95_headers_ms": percentile([r["headers_ms"] for r in rows], 95),
|
||||
"routes": sorted(aggregates, key=lambda r: r["max_ms"], reverse=True)[:40],
|
||||
"slow_requests": sorted(rows, key=lambda r: r["total_ms"], reverse=True)[:30],
|
||||
"active": [{**r, "elapsed_ms": round((now - r["started"]) * 1000, 1)} for r in list(self.active.values())[:100]]}
|
||||
|
||||
def worker_status(self):
|
||||
"""Bounded status payload for the main process' 5-second health probe."""
|
||||
summary = self.summary()
|
||||
return {
|
||||
"count": summary["count"],
|
||||
"errors": summary["errors"],
|
||||
"p95_headers_ms": summary["p95_headers_ms"],
|
||||
"routes": summary["routes"][:8],
|
||||
"active_count": len(self.active),
|
||||
}
|
||||
|
||||
|
||||
class RequestTelemetryMiddleware:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self._active_requests = 0
|
||||
self._active_by_path = {}
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
recorder = getattr(scope["app"].state, "request_recorder", None)
|
||||
path = scope.get("path", "")
|
||||
if not path.startswith("/api/") or path.startswith(("/api/system-settings/worker-pool", "/api/health")):
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
record_request = recorder is not None and recorder.enabled and len(recorder.active) < 2000
|
||||
begin = time.perf_counter()
|
||||
request_id = uuid.uuid4().hex[:12]
|
||||
route_key = route_name(path)
|
||||
self._active_requests += 1
|
||||
self._active_by_path[route_key] = self._active_by_path.get(route_key, 0) + 1
|
||||
api_inflight_start = self._active_requests
|
||||
route_inflight_start = self._active_by_path[route_key]
|
||||
record = {"id": request_id, "started": time.time(), "method": scope["method"], "path": route_name(path),
|
||||
"status": 499, "headers_ms": None, "first_body_ms": None, "stream": False} if record_request else None
|
||||
if record is not None:
|
||||
recorder.active[record["id"]] = record
|
||||
state = scope.setdefault("state", {})
|
||||
state["diagnostic_request_id"] = request_id
|
||||
state["perf_api_inflight_start"] = api_inflight_start
|
||||
state["perf_route_inflight_start"] = route_inflight_start
|
||||
request_context_token = None
|
||||
try:
|
||||
from deerflow.persistence.engine import bind_performance_request
|
||||
|
||||
request_context_token = bind_performance_request(request_id)
|
||||
except Exception:
|
||||
logger.debug("PERF request ID could not be attached to the SQL context", exc_info=True)
|
||||
|
||||
async def measured_send(message):
|
||||
if message["type"] == "http.response.start":
|
||||
headers_ms = round((time.perf_counter() - begin) * 1000, 2)
|
||||
headers = list(message.get("headers", []))
|
||||
is_stream = any(k.lower() == b"content-type" and b"text/event-stream" in v for k, v in headers)
|
||||
state["perf_headers_ms"] = headers_ms
|
||||
state["perf_stream"] = is_stream
|
||||
state["perf_status"] = message["status"]
|
||||
if record is not None:
|
||||
record["status"] = message["status"]
|
||||
record["headers_ms"] = headers_ms
|
||||
record["stream"] = is_stream
|
||||
# Record existing operation phases without query text or headers.
|
||||
phases = next((v.decode("latin-1")[:1000] for k, v in headers if k.lower() == b"server-timing"), "")
|
||||
if record is not None:
|
||||
record["phases"] = re.sub(r"[^a-zA-Z0-9_.,;= -]", "", phases)
|
||||
headers.append((b"x-deerflow-request-id", request_id.encode()))
|
||||
message = {**message, "headers": headers}
|
||||
elif message["type"] == "http.response.body" and message.get("body") and state.get("perf_first_body_ms") is None:
|
||||
state["perf_first_body_ms"] = round((time.perf_counter() - begin) * 1000, 2)
|
||||
if record is not None:
|
||||
record["first_body_ms"] = state["perf_first_body_ms"]
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, measured_send)
|
||||
except BaseException as exc:
|
||||
cancelled = type(exc).__name__.lower() in {"cancelled", "cancellederror"} or "cancelled" in type(exc).__name__.lower()
|
||||
if record is not None and record["headers_ms"] is None:
|
||||
record["status"] = 499 if cancelled else 500
|
||||
state["perf_exception_type"] = type(exc).__name__
|
||||
state["perf_cancelled"] = cancelled
|
||||
if record is None or record["headers_ms"] is None:
|
||||
state["perf_status"] = 499 if cancelled else 500
|
||||
raise
|
||||
finally:
|
||||
total_ms = round((time.perf_counter() - begin) * 1000, 2)
|
||||
route = scope.get("route")
|
||||
route_path = getattr(route, "path", None) or route_name(path)
|
||||
route_path = route_path[:240]
|
||||
headers_ms = record.get("headers_ms") if record is not None else state.get("perf_headers_ms")
|
||||
is_stream = bool(state.get("perf_stream", record.get("stream") if record is not None else False))
|
||||
status = record.get("status", 500) if record is not None else state.get("perf_status", 500)
|
||||
exception_type = state.get("perf_exception_type")
|
||||
node = state.get("worker_node", "main")
|
||||
trace = state.get("perf_trace")
|
||||
threshold = should_log_request(
|
||||
elapsed_ms=total_ms,
|
||||
headers_ms=headers_ms,
|
||||
first_body_ms=state.get("perf_first_body_ms"),
|
||||
is_stream=is_stream,
|
||||
)
|
||||
verbose = verbose_request_logging() and is_verbose_path(path)
|
||||
cancelled = state.get("perf_cancelled", False)
|
||||
concurrency_detail = api_inflight_start >= 8
|
||||
detailed = threshold or verbose or status >= 500 or (exception_type is not None and not cancelled) or concurrency_detail
|
||||
if detailed:
|
||||
endpoint_ms = trace.get("endpoint_ms") if isinstance(trace, dict) else None
|
||||
pool_snapshot = _database_pool_snapshot()
|
||||
level = logging.ERROR if not cancelled and (status >= 500 or exception_type is not None) else logging.WARNING if threshold else logging.INFO
|
||||
logger.log(
|
||||
level,
|
||||
"PERF http_request id=%s method=%s path=%s status=%s total_ms=%.2f headers_ms=%s first_body_ms=%s stream=%s "
|
||||
"api_inflight_start=%s route_inflight_start=%s api_inflight_end=%s route_inflight_end=%s db_pool=%s node=%s "
|
||||
"worker_access_ms=%s worker_upstream_headers_ms=%s worker_fallback=%s event_loop_lag_ms=%s "
|
||||
"exception_type=%s endpoint=%s endpoint_ms=%s outside_handler_ms=%s stages=%s",
|
||||
request_id,
|
||||
scope["method"],
|
||||
route_path,
|
||||
status,
|
||||
total_ms,
|
||||
headers_ms,
|
||||
state.get("perf_first_body_ms"),
|
||||
is_stream,
|
||||
api_inflight_start,
|
||||
route_inflight_start,
|
||||
self._active_requests,
|
||||
self._active_by_path.get(route_key, 0),
|
||||
pool_snapshot,
|
||||
node,
|
||||
state.get("worker_access_ms"),
|
||||
state.get("worker_upstream_headers_ms"),
|
||||
state.get("worker_pool_fallback_reason"),
|
||||
getattr(scope["app"].state, "perf_event_loop_lag_ms", None),
|
||||
exception_type,
|
||||
trace.get("endpoint") if isinstance(trace, dict) else None,
|
||||
endpoint_ms,
|
||||
round(max(0.0, headers_ms - endpoint_ms), 2) if headers_ms is not None and endpoint_ms is not None else None,
|
||||
{"stages": trace.get("stages"), "details": trace.get("details")} if isinstance(trace, dict) else None,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"PERF http_request id=%s method=%s path=%s status=%s total_ms=%.2f headers_ms=%s api_inflight_start=%s route_inflight_start=%s api_inflight_end=%s route_inflight_end=%s node=%s fallback=%s exception_type=%s",
|
||||
request_id,
|
||||
scope["method"],
|
||||
route_path,
|
||||
status,
|
||||
total_ms,
|
||||
headers_ms,
|
||||
api_inflight_start,
|
||||
route_inflight_start,
|
||||
self._active_requests,
|
||||
self._active_by_path.get(route_key, 0),
|
||||
node,
|
||||
state.get("worker_pool_fallback_reason"),
|
||||
exception_type,
|
||||
)
|
||||
state["perf_status"] = status
|
||||
if record is not None:
|
||||
record["total_ms"] = total_ms
|
||||
record["node"] = node
|
||||
record["access_ms"] = state.get("worker_access_ms")
|
||||
record["upstream_headers_ms"] = state.get("worker_upstream_headers_ms")
|
||||
record["fallback_reason"] = state.get("worker_pool_fallback_reason")
|
||||
if record is not None:
|
||||
record["path"] = route_path
|
||||
recorder.finish(record)
|
||||
if request_context_token is not None:
|
||||
try:
|
||||
from deerflow.persistence.engine import reset_performance_request
|
||||
|
||||
reset_performance_request(request_context_token)
|
||||
except Exception:
|
||||
logger.debug("PERF request ID could not be cleared from the SQL context", exc_info=True)
|
||||
self._active_requests = max(0, self._active_requests - 1)
|
||||
remaining_for_path = self._active_by_path.get(route_key, 0) - 1
|
||||
if remaining_for_path > 0:
|
||||
self._active_by_path[route_key] = remaining_for_path
|
||||
else:
|
||||
self._active_by_path.pop(route_key, None)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Dedicated single-process online execution entrypoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import os
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from app.gateway.app import create_app
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.runtime_executor import configure_runtime_executor
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
from .admission import Admission
|
||||
from .manager import WorkerRunManager
|
||||
from .preflight import shared_storage_configured, validate_sqlite_mount
|
||||
from .proxy import THREAD_PATH
|
||||
from .refresh import SharedSkillRefresh
|
||||
|
||||
|
||||
async def _measure_worker_event_loop_lag(app):
|
||||
loop = asyncio.get_running_loop()
|
||||
while True:
|
||||
if not getattr(app.state, "worker_diagnostics_enabled", False):
|
||||
app.state.worker_event_loop_lag_ms = 0.0
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
started = loop.time()
|
||||
await asyncio.sleep(1)
|
||||
app.state.worker_event_loop_lag_ms = round(max(0.0, loop.time() - started - 1) * 1000, 2)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def worker_lifespan(app):
|
||||
from .revision import code_revision
|
||||
app.state.worker_process_started = time.time()
|
||||
app.state.worker_runtime_revision = code_revision()
|
||||
config = get_app_config()
|
||||
from .runtime import verify_storage_dependencies
|
||||
|
||||
verify_storage_dependencies(config)
|
||||
if not shared_storage_configured(config):
|
||||
raise RuntimeError("Managed workers require persistent PostgreSQL or SQLite chat storage")
|
||||
if "AioSandboxProvider" in str(getattr(config.sandbox, "use", "")):
|
||||
raise RuntimeError("Managed online workers cannot launch Docker sandboxes")
|
||||
validate_sqlite_mount(config)
|
||||
# Shared event records follow conversations across drained containers.
|
||||
config = config.model_copy(
|
||||
update={
|
||||
"run_events": config.run_events.model_copy(update={"backend": "db"}),
|
||||
"database": config.database.model_copy(update={"pool_size": int(os.environ.get("DEER_FLOW_WORKER_DB_POOL_SIZE", "4")), "max_overflow": 0}),
|
||||
}
|
||||
)
|
||||
app.state.config = config
|
||||
app.state.worker_skill_refresh = SharedSkillRefresh()
|
||||
app.state.worker_deferred_admission = True
|
||||
app.state.worker_admission = Admission(int(os.environ.get("DEER_FLOW_WORKER_CONCURRENCY", "4")), int(os.environ.get("DEER_FLOW_WORKER_QUEUE_LIMIT", "128")), float(os.environ.get("DEER_FLOW_WORKER_QUEUE_TIMEOUT", "60")))
|
||||
# Each execution container keeps its own bounded rolling request metrics;
|
||||
# the controller reads a compact summary with the normal status probe.
|
||||
from .telemetry import RequestRecorder
|
||||
|
||||
app.state.request_recorder = RequestRecorder()
|
||||
app.state.worker_diagnostics_enabled = False
|
||||
app.state.worker_event_loop_lag_ms = 0.0
|
||||
await configure_runtime_executor(config.runtime_performance)
|
||||
async with langgraph_runtime(app, background=False):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
|
||||
if isinstance(app.state.checkpointer, InMemorySaver) or isinstance(app.state.store, InMemoryStore):
|
||||
raise RuntimeError("Managed worker refuses an in-memory persistence fallback")
|
||||
close_manager = getattr(app.state.run_manager, "aclose", None)
|
||||
if close_manager is not None:
|
||||
await close_manager()
|
||||
app.state.run_manager = WorkerRunManager(store=app.state.run_store)
|
||||
retention = getattr(config, "runtime_contention", None)
|
||||
if retention is not None and hasattr(app.state.run_manager, "start_retention"):
|
||||
app.state.run_manager.start_retention(retention_seconds=retention.completed_run_retention_seconds, interval_seconds=retention.run_sweep_seconds)
|
||||
lag_task = asyncio.create_task(_measure_worker_event_loop_lag(app))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
lag_task.cancel()
|
||||
await asyncio.gather(lag_task, return_exceptions=True)
|
||||
|
||||
|
||||
class InternalOnly:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
expected = os.environ.get("DEER_FLOW_WORKER_TOKEN", "")
|
||||
token = dict(scope["headers"]).get(b"x-deerflow-worker-token", b"").decode()
|
||||
if not expected or not hmac.compare_digest(token, expected):
|
||||
return await JSONResponse({"detail": "Forbidden"}, 403)(scope, receive, send)
|
||||
if scope["path"] == "/internal/worker-status":
|
||||
gate = scope["app"].state.worker_admission
|
||||
recorder = getattr(scope["app"].state, "request_recorder", None)
|
||||
diagnostics_enabled = dict(scope["headers"]).get(b"x-deerflow-diagnostics-enabled") == b"true"
|
||||
scope["app"].state.worker_diagnostics_enabled = diagnostics_enabled
|
||||
if recorder is not None:
|
||||
recorder.enabled = diagnostics_enabled
|
||||
http = recorder.worker_status() if recorder is not None else None
|
||||
return await JSONResponse({"ready": True, "active": gate.active + gate.waiting, "running": gate.active, "waiting": gate.waiting, "capacity": gate.capacity,
|
||||
"process_started_at": scope["app"].state.worker_process_started,
|
||||
"runtime_revision": scope["app"].state.worker_runtime_revision,
|
||||
"event_loop_lag_ms": getattr(scope["app"].state, "worker_event_loop_lag_ms", None),
|
||||
"diagnostics_enabled": diagnostics_enabled,
|
||||
"http": http})(scope, receive, send)
|
||||
active_prefix = "/internal/worker-thread-active/"
|
||||
if scope["path"].startswith(active_prefix):
|
||||
from urllib.parse import unquote
|
||||
|
||||
thread_id = unquote(scope["path"][len(active_prefix):])
|
||||
active = scope["app"].state.run_manager.has_active_thread(thread_id)
|
||||
return await JSONResponse({"active": active})(scope, receive, send)
|
||||
match = THREAD_PATH.fullmatch(scope["path"])
|
||||
if not match:
|
||||
return await JSONResponse({"detail": "Worker endpoint not available"}, 404)(scope, receive, send)
|
||||
scope.setdefault("state", {})["worker_node"] = os.environ.get("HOSTNAME", "worker")
|
||||
from deerflow.runtime.thread_paths import aresolve_path_user_id
|
||||
|
||||
await aresolve_path_user_id(match[1])
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
app = create_app()
|
||||
app.router.lifespan_context = worker_lifespan
|
||||
app.add_middleware(InternalOnly)
|
||||
Reference in new issue
Block a user