4783 lines
207 KiB
Python
4783 lines
207 KiB
Python
#!/usr/bin/env python3
|
||
"""Standalone evidence-first latency diagnostics for DeerFlow HTTP deployments.
|
||
|
||
The script is intentionally standard-library-only so it can be copied to a
|
||
Linux deployment host and run with the system Python. It combines:
|
||
|
||
* fresh-connection HTTP timing (DNS, TCP, TLS, response wait, first body byte,
|
||
body read, total), including bounded concurrency and direct-backend
|
||
comparisons;
|
||
* Linux host, process, socket/backlog, file-descriptor, pressure and Docker
|
||
snapshots;
|
||
* read-only Nginx capacity/config inspection and log-pattern aggregation;
|
||
* optional read-only MySQL status/process/lock checks through the ``mysql``
|
||
command-line client; and
|
||
* optional DeerFlow administrator concurrency endpoints.
|
||
|
||
Security invariants:
|
||
|
||
* URL query values and userinfo are always redacted from console/JSON output.
|
||
* Request/response bodies and secret header values are never printed or saved.
|
||
* Successful login and agent POSTs can mutate session/run state, so unsafe
|
||
methods are never replayed unless ``--allow-unsafe-replay`` is explicit.
|
||
* TLS verification is enabled by default. ``--insecure`` is explicit and is
|
||
recorded as a high-severity caveat in the report.
|
||
|
||
This tool reports evidence and cautious localization. A high response wait is
|
||
not, by itself, proof of a slow database, Nginx, or an LLM provider.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import concurrent.futures
|
||
import contextlib
|
||
import datetime as dt
|
||
import getpass
|
||
import glob
|
||
import hashlib
|
||
import http.client
|
||
import io
|
||
import ipaddress
|
||
import json
|
||
import math
|
||
import os
|
||
import platform
|
||
import re
|
||
import shutil
|
||
import socket
|
||
import ssl
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from collections import Counter, defaultdict
|
||
from collections.abc import Iterable, Mapping, Sequence
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
REPORT_VERSION = 1
|
||
USER_AGENT = "deerflow-latency-diagnosis/1"
|
||
SAFE_METHODS = {"GET", "HEAD", "OPTIONS"}
|
||
SECRET_HEADER_RE = re.compile(
|
||
r"(?:authorization|proxy-authorization|cookie|set-cookie|token|api[-_]?key|secret|password|csrf|session|credential)",
|
||
re.IGNORECASE,
|
||
)
|
||
SECRET_QUERY_RE = re.compile(
|
||
r"(?:token|auth|password|passwd|secret|key|ticket|session|cookie|credential)",
|
||
re.IGNORECASE,
|
||
)
|
||
MAX_HEADER_BYTES = 1024 * 1024
|
||
MAX_REQUEST_HEADER_BYTES = 64 * 1024
|
||
MAX_COMMAND_OUTPUT = 4 * 1024 * 1024
|
||
UTC = getattr(dt, "UTC", dt.timezone.utc) # noqa: UP017 - keep the standalone tool usable on Python 3.9
|
||
HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$")
|
||
|
||
|
||
@dataclass
|
||
class ProbeOptions:
|
||
method: str = "GET"
|
||
headers: dict[str, str] = field(default_factory=dict)
|
||
body: bytes = b""
|
||
connect_timeout: float = 5.0
|
||
first_byte_timeout: float = 30.0
|
||
read_timeout: float = 5.0
|
||
total_timeout: float = 120.0
|
||
max_response_bytes: int = 1024 * 1024
|
||
read_mode: str = "auto"
|
||
insecure: bool = False
|
||
ca_file: str | None = None
|
||
follow_redirects: bool = False
|
||
max_redirects: int = 3
|
||
proxy: str | None = None
|
||
use_env_proxy: bool = False
|
||
ip_family: str = "auto"
|
||
|
||
|
||
class ProbeFailure(RuntimeError):
|
||
"""Transport failure with the phase that failed."""
|
||
|
||
def __init__(self, phase: str, message: str):
|
||
super().__init__(message)
|
||
self.phase = phase
|
||
|
||
|
||
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||
"""Return 30x to the caller instead of forwarding credentials implicitly."""
|
||
|
||
def redirect_request(
|
||
self,
|
||
req: urllib.request.Request,
|
||
fp: Any,
|
||
code: int,
|
||
msg: str,
|
||
headers: Any,
|
||
newurl: str,
|
||
) -> None:
|
||
return None
|
||
|
||
|
||
class _SocketBodyReader:
|
||
"""Bounded socket reader used by HTTP body framing parsers."""
|
||
|
||
def __init__(
|
||
self,
|
||
sock: socket.socket,
|
||
initial: bytes,
|
||
*,
|
||
idle_timeout: float,
|
||
deadline: float,
|
||
idle_timeout_phase: str = "body_idle_timeout",
|
||
):
|
||
self.sock = sock
|
||
self.buffer = bytearray(initial)
|
||
self.idle_timeout = max(0.1, idle_timeout)
|
||
self.deadline = deadline
|
||
self.idle_timeout_phase = idle_timeout_phase
|
||
self.eof = False
|
||
|
||
def _fill(self) -> None:
|
||
if self.eof:
|
||
return
|
||
remaining = self.deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("body_total_timeout", "overall request timeout expired while reading response body")
|
||
self.sock.settimeout(min(self.idle_timeout, remaining))
|
||
try:
|
||
chunk = self.sock.recv(65536)
|
||
except (TimeoutError, socket.timeout) as exc: # noqa: UP041 - Python 3.9/Windows may not alias these
|
||
phase = "body_total_timeout" if time.perf_counter() >= self.deadline else self.idle_timeout_phase
|
||
raise ProbeFailure(phase, f"response body made no progress for {min(self.idle_timeout, remaining):.3f}s") from exc
|
||
except OSError as exc:
|
||
raise ProbeFailure("body_read", _safe_error(exc)) from exc
|
||
if not chunk:
|
||
self.eof = True
|
||
return
|
||
self.buffer.extend(chunk)
|
||
|
||
def read_some(self, maximum: int) -> bytes:
|
||
while not self.buffer and not self.eof:
|
||
self._fill()
|
||
if not self.buffer:
|
||
return b""
|
||
size = min(maximum, len(self.buffer))
|
||
output = bytes(self.buffer[:size])
|
||
del self.buffer[:size]
|
||
return output
|
||
|
||
def read_exact(self, size: int) -> bytes:
|
||
while len(self.buffer) < size and not self.eof:
|
||
self._fill()
|
||
if len(self.buffer) < size:
|
||
raise ProbeFailure("body_framing", "response ended before the declared body framing completed")
|
||
output = bytes(self.buffer[:size])
|
||
del self.buffer[:size]
|
||
return output
|
||
|
||
def read_line(self, limit: int = 8192) -> bytes:
|
||
while True:
|
||
marker = self.buffer.find(b"\r\n")
|
||
if marker >= 0:
|
||
if marker > limit:
|
||
raise ProbeFailure("body_framing", "HTTP chunk framing line exceeded 8192 bytes")
|
||
output = bytes(self.buffer[:marker])
|
||
del self.buffer[: marker + 2]
|
||
return output
|
||
if len(self.buffer) > limit:
|
||
raise ProbeFailure("body_framing", "HTTP chunk framing line exceeded 8192 bytes")
|
||
if self.eof:
|
||
raise ProbeFailure("body_framing", "response ended inside HTTP chunk framing")
|
||
self._fill()
|
||
|
||
|
||
def utc_now_iso() -> str:
|
||
return dt.datetime.now(UTC).isoformat()
|
||
|
||
|
||
def monotonic_ms(start: float, end: float | None = None) -> float:
|
||
return round(((end if end is not None else time.perf_counter()) - start) * 1000.0, 3)
|
||
|
||
|
||
def _safe_error(exc: BaseException, limit: int = 300) -> str:
|
||
"""Return a bounded error string with obvious credentials redacted."""
|
||
text = f"{type(exc).__name__}: {exc}"
|
||
text = re.sub(
|
||
r"https?://[^\s\]\[(){}<>'\"]+",
|
||
lambda match: sanitize_url(match.group(0)),
|
||
text,
|
||
flags=re.IGNORECASE,
|
||
)
|
||
text = re.sub(
|
||
r"(?i)(authorization|cookie|token|password|secret|api[-_]?key|csrf|session|credential)\s*[:=]\s*[^\s,;]+",
|
||
r"\1=<redacted>",
|
||
text,
|
||
)
|
||
text = re.sub(r"(?i)://[^/@\s]+@", "://<redacted>@", text)
|
||
return text[:limit]
|
||
|
||
|
||
def sanitize_url(url: str) -> str:
|
||
"""Redact userinfo and every query value while retaining routing shape."""
|
||
try:
|
||
parsed = urllib.parse.urlsplit(url)
|
||
except ValueError:
|
||
return "<invalid-url>"
|
||
if not parsed.scheme or not parsed.netloc:
|
||
return "<invalid-url>"
|
||
hostname = parsed.hostname or ""
|
||
try:
|
||
ip = ipaddress.ip_address(hostname)
|
||
rendered_host = f"[{ip}]" if ip.version == 6 else str(ip)
|
||
except ValueError:
|
||
rendered_host = hostname
|
||
try:
|
||
parsed_port = parsed.port
|
||
except ValueError:
|
||
return "<invalid-url>"
|
||
if parsed_port is not None:
|
||
rendered_host = f"{rendered_host}:{parsed_port}"
|
||
safe_query: list[tuple[str, str]] = []
|
||
for index, query_field in enumerate(parsed.query.split("&")):
|
||
if not query_field:
|
||
continue
|
||
raw_key, separator, _raw_value = query_field.partition("=")
|
||
key = urllib.parse.unquote_plus(raw_key)
|
||
key_is_safe = bool(separator and len(key) <= 64 and re.fullmatch(r"[A-Za-z0-9_.~-]+", key))
|
||
safe_key = key if key_is_safe else f"param_{index + 1}"
|
||
marker = "<secret>" if SECRET_QUERY_RE.search(key) else "<redacted>"
|
||
safe_query.append((safe_key, marker))
|
||
query = urllib.parse.urlencode(safe_query)
|
||
return urllib.parse.urlunsplit((parsed.scheme, rendered_host, parsed.path or "/", query, ""))
|
||
|
||
|
||
def sanitize_origin(url: str) -> str:
|
||
try:
|
||
parsed = urllib.parse.urlsplit(url)
|
||
parsed_port = parsed.port
|
||
except ValueError:
|
||
return "<invalid-origin>"
|
||
hostname = parsed.hostname or ""
|
||
try:
|
||
ip = ipaddress.ip_address(hostname)
|
||
rendered_host = f"[{ip}]" if ip.version == 6 else str(ip)
|
||
except ValueError:
|
||
rendered_host = hostname
|
||
default_port = 443 if parsed.scheme == "https" else 80
|
||
if parsed_port and parsed_port != default_port:
|
||
rendered_host = f"{rendered_host}:{parsed_port}"
|
||
return f"{parsed.scheme}://{rendered_host}"
|
||
|
||
|
||
def _origin_tuple(url: str) -> tuple[str, str, int]:
|
||
parsed = urllib.parse.urlsplit(url)
|
||
scheme = parsed.scheme.lower()
|
||
if scheme not in {"http", "https"} or not parsed.hostname:
|
||
raise ValueError("URL must use http:// or https:// and include a host")
|
||
return scheme, parsed.hostname.lower(), parsed.port or (443 if scheme == "https" else 80)
|
||
|
||
|
||
def _host_is_loopback(hostname: str) -> bool:
|
||
normalized = hostname.rstrip(".").lower()
|
||
if normalized == "localhost":
|
||
return True
|
||
try:
|
||
return ipaddress.ip_address(normalized).is_loopback
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
def _url_has_secret_query(url: str) -> bool:
|
||
return any(
|
||
bool(value) and bool(SECRET_QUERY_RE.search(key))
|
||
for key, value in urllib.parse.parse_qsl(
|
||
urllib.parse.urlsplit(url).query,
|
||
keep_blank_values=True,
|
||
)
|
||
)
|
||
|
||
|
||
def _header_is_secret(name: str) -> bool:
|
||
return bool(SECRET_HEADER_RE.search(name))
|
||
|
||
|
||
def _validate_header(name: str, value: str) -> None:
|
||
if not HEADER_NAME_RE.fullmatch(name):
|
||
raise ValueError(f"Invalid header name: {name!r}")
|
||
if any((ord(ch) < 32 and ch != "\t") or ord(ch) == 127 or ord(ch) > 255 for ch in value):
|
||
raise ValueError(f"Header {name!r} contains an illegal control or non-Latin-1 character")
|
||
if len(value.encode("iso-8859-1")) > 16 * 1024:
|
||
raise ValueError(f"Header {name!r} exceeds 16 KiB")
|
||
|
||
|
||
def parse_header_argument(raw: str) -> tuple[str, str]:
|
||
name, separator, value = raw.partition(":")
|
||
if not separator:
|
||
raise ValueError("Header must use NAME:VALUE")
|
||
name = name.strip()
|
||
value = value.strip()
|
||
_validate_header(name, value)
|
||
return name, value
|
||
|
||
|
||
def parse_header_env_argument(raw: str) -> tuple[str, str]:
|
||
name, separator, env_name = raw.partition("=")
|
||
if not separator or not name.strip() or not env_name.strip():
|
||
raise ValueError("Environment header must use HEADER=ENV_NAME")
|
||
value = os.environ.get(env_name.strip())
|
||
if value is None:
|
||
raise ValueError(f"Environment variable {env_name.strip()} is not set")
|
||
name = name.strip()
|
||
_validate_header(name, value)
|
||
return name, value
|
||
|
||
|
||
def parse_server_timing(raw: str | None) -> list[dict[str, Any]]:
|
||
"""Parse only metric name/duration; deliberately discard descriptions."""
|
||
if not raw:
|
||
return []
|
||
metrics: list[dict[str, Any]] = []
|
||
for part in raw.split(","):
|
||
segments = [segment.strip() for segment in part.split(";") if segment.strip()]
|
||
if not segments:
|
||
continue
|
||
name = re.sub(r"[^A-Za-z0-9_.:-]", "", segments[0])[:80]
|
||
if not name:
|
||
continue
|
||
item: dict[str, Any] = {"name": name}
|
||
for segment in segments[1:]:
|
||
key, separator, value = segment.partition("=")
|
||
if separator and key.strip().lower() == "dur":
|
||
try:
|
||
duration = float(value.strip().strip('"'))
|
||
except ValueError:
|
||
continue
|
||
if math.isfinite(duration) and duration >= 0:
|
||
item["duration_ms"] = round(duration, 3)
|
||
metrics.append(item)
|
||
return metrics
|
||
|
||
|
||
def percentile(values: Sequence[float], percentile_value: float) -> float | None:
|
||
if not values:
|
||
return None
|
||
ordered = sorted(float(value) for value in values)
|
||
if len(ordered) == 1:
|
||
return round(ordered[0], 3)
|
||
rank = (len(ordered) - 1) * percentile_value
|
||
lower = math.floor(rank)
|
||
upper = math.ceil(rank)
|
||
if lower == upper:
|
||
return round(ordered[lower], 3)
|
||
weight = rank - lower
|
||
return round(ordered[lower] * (1.0 - weight) + ordered[upper] * weight, 3)
|
||
|
||
|
||
def summarize_numbers(values: Iterable[float | int | None]) -> dict[str, float | int | None]:
|
||
clean = [float(value) for value in values if value is not None and math.isfinite(float(value))]
|
||
if not clean:
|
||
return {"samples": 0, "min": None, "p50": None, "p95": None, "max": None, "avg": None}
|
||
return {
|
||
"samples": len(clean),
|
||
"min": round(min(clean), 3),
|
||
"p50": percentile(clean, 0.50),
|
||
"p95": percentile(clean, 0.95),
|
||
"max": round(max(clean), 3),
|
||
"avg": round(statistics.fmean(clean), 3),
|
||
}
|
||
|
||
|
||
def _select_proxy(url: str, options: ProbeOptions) -> str | None:
|
||
if options.proxy:
|
||
return options.proxy
|
||
if not options.use_env_proxy:
|
||
return None
|
||
parsed = urllib.parse.urlsplit(url)
|
||
hostname = parsed.hostname or ""
|
||
bypass_targets = [hostname] if hostname else []
|
||
if parsed.port is not None:
|
||
try:
|
||
rendered_bypass_host = f"[{hostname}]" if ipaddress.ip_address(hostname).version == 6 else hostname
|
||
except ValueError:
|
||
rendered_bypass_host = hostname
|
||
bypass_targets.append(f"{rendered_bypass_host}:{parsed.port}")
|
||
if any(target and urllib.request.proxy_bypass(target) for target in dict.fromkeys(bypass_targets)):
|
||
return None
|
||
proxies = urllib.request.getproxies()
|
||
return proxies.get(parsed.scheme.lower())
|
||
|
||
|
||
def _address_family(value: str) -> int:
|
||
if value == "ipv4":
|
||
return socket.AF_INET
|
||
if value == "ipv6":
|
||
return socket.AF_INET6
|
||
return socket.AF_UNSPEC
|
||
|
||
|
||
def _connect_socket(
|
||
host: str,
|
||
port: int,
|
||
*,
|
||
timeout: float,
|
||
family: str,
|
||
) -> tuple[socket.socket, dict[str, Any]]:
|
||
deadline = time.perf_counter() + timeout
|
||
dns_started = time.perf_counter()
|
||
resolution: dict[str, Any] = {}
|
||
resolved = threading.Event()
|
||
|
||
def resolve() -> None:
|
||
try:
|
||
resolution["infos"] = socket.getaddrinfo(
|
||
host,
|
||
port,
|
||
family=_address_family(family),
|
||
type=socket.SOCK_STREAM,
|
||
)
|
||
except Exception as exc: # captured and re-raised in the caller thread
|
||
resolution["error"] = exc
|
||
finally:
|
||
resolved.set()
|
||
|
||
resolver = threading.Thread(target=resolve, name="deerflow-dns-probe", daemon=True)
|
||
resolver.start()
|
||
if not resolved.wait(timeout):
|
||
raise ProbeFailure("dns", f"timed out after {timeout:.3f}s")
|
||
error = resolution.get("error")
|
||
if error is not None:
|
||
if isinstance(error, socket.gaierror):
|
||
raise ProbeFailure("dns", _safe_error(error)) from error
|
||
raise ProbeFailure("dns", _safe_error(error))
|
||
infos = resolution.get("infos") or []
|
||
dns_ms = monotonic_ms(dns_started)
|
||
if not infos:
|
||
raise ProbeFailure("dns", "DNS returned no usable address")
|
||
|
||
attempts: list[dict[str, Any]] = []
|
||
last_error: BaseException | None = None
|
||
tcp_started = time.perf_counter()
|
||
for af, socktype, protocol, _canonname, sockaddr in infos:
|
||
remaining = deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("connect", f"DNS and TCP connection exceeded {timeout:.3f}s")
|
||
sock = socket.socket(af, socktype, protocol)
|
||
sock.settimeout(remaining)
|
||
started = time.perf_counter()
|
||
try:
|
||
sock.connect(sockaddr)
|
||
except OSError as exc:
|
||
last_error = exc
|
||
attempts.append(
|
||
{
|
||
"family": "ipv6" if af == socket.AF_INET6 else "ipv4",
|
||
"address": str(sockaddr[0]),
|
||
"connect_ms": monotonic_ms(started),
|
||
"outcome": type(exc).__name__,
|
||
}
|
||
)
|
||
sock.close()
|
||
continue
|
||
connect_ms = monotonic_ms(started)
|
||
attempts.append(
|
||
{
|
||
"family": "ipv6" if af == socket.AF_INET6 else "ipv4",
|
||
"address": str(sockaddr[0]),
|
||
"connect_ms": connect_ms,
|
||
"outcome": "connected",
|
||
}
|
||
)
|
||
return sock, {
|
||
"dns_ms": dns_ms,
|
||
"connect_ms": monotonic_ms(tcp_started),
|
||
"chosen_address_connect_ms": connect_ms,
|
||
"peer_ip": str(sockaddr[0]),
|
||
"connect_attempts": attempts,
|
||
}
|
||
raise ProbeFailure("connect", _safe_error(last_error or OSError("all addresses failed")))
|
||
|
||
|
||
def _recv_header_block(
|
||
sock: socket.socket,
|
||
timeout: float,
|
||
*,
|
||
initial: bytes = b"",
|
||
) -> tuple[bytes, bytes, float]:
|
||
deadline = time.perf_counter() + timeout
|
||
buffer = bytearray(initial)
|
||
first_byte_at: float | None = time.perf_counter() if initial else None
|
||
while b"\r\n\r\n" not in buffer:
|
||
remaining = deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("first_byte", f"response headers exceeded {timeout:.3f}s total")
|
||
sock.settimeout(remaining)
|
||
try:
|
||
chunk = sock.recv(65536)
|
||
except (TimeoutError, socket.timeout) as exc: # noqa: UP041 - standalone Python compatibility
|
||
raise ProbeFailure("first_byte", f"timed out after {timeout:.3f}s") from exc
|
||
except OSError as exc:
|
||
raise ProbeFailure("first_byte", _safe_error(exc)) from exc
|
||
if not chunk:
|
||
raise ProbeFailure("http_headers", "connection closed before response headers completed")
|
||
if first_byte_at is None:
|
||
first_byte_at = time.perf_counter()
|
||
buffer.extend(chunk)
|
||
if len(buffer) > MAX_HEADER_BYTES and b"\r\n\r\n" not in buffer:
|
||
raise ProbeFailure("http_headers", "response headers exceeded 1 MiB")
|
||
head, body = bytes(buffer).split(b"\r\n\r\n", 1)
|
||
if len(head) > MAX_HEADER_BYTES:
|
||
raise ProbeFailure("http_headers", "response headers exceeded 1 MiB")
|
||
return head, body, first_byte_at or time.perf_counter()
|
||
|
||
|
||
def _parse_header_block(head: bytes) -> tuple[int, str, dict[str, str]]:
|
||
first_line, separator, remainder = head.partition(b"\r\n")
|
||
if not separator:
|
||
raise ProbeFailure("http_headers", "malformed HTTP status line")
|
||
try:
|
||
status_text = first_line.decode("iso-8859-1")
|
||
status_parts = status_text.split(" ", 2)
|
||
if len(status_parts) < 2:
|
||
raise ValueError("invalid status line")
|
||
version, status_raw = status_parts[:2]
|
||
reason = status_parts[2] if len(status_parts) == 3 else ""
|
||
if not re.fullmatch(r"HTTP/\d+\.\d+", version):
|
||
raise ValueError("invalid protocol")
|
||
if not re.fullmatch(r"\d{3}", status_raw):
|
||
raise ValueError("invalid status code")
|
||
status = int(status_raw)
|
||
parsed_headers = http.client.parse_headers(io.BytesIO(remainder + b"\r\n\r\n"))
|
||
except (ValueError, http.client.HTTPException) as exc:
|
||
raise ProbeFailure("http_headers", _safe_error(exc)) from exc
|
||
headers: dict[str, str] = {}
|
||
for name in parsed_headers.keys():
|
||
values = parsed_headers.get_all(name) or []
|
||
headers[name.lower()] = ", ".join(values)
|
||
return status, reason[:100], headers
|
||
|
||
|
||
def _safe_response_headers(headers: Mapping[str, str]) -> dict[str, str]:
|
||
allowed = {
|
||
"server",
|
||
"via",
|
||
"content-type",
|
||
"content-length",
|
||
"transfer-encoding",
|
||
"connection",
|
||
"x-accel-buffering",
|
||
"x-cache",
|
||
"retry-after",
|
||
}
|
||
return {name: str(value)[:300] for name, value in headers.items() if name.lower() in allowed and not _header_is_secret(name)}
|
||
|
||
|
||
def _proxy_authorization(parsed_proxy: urllib.parse.SplitResult) -> str | None:
|
||
if parsed_proxy.username is None:
|
||
return None
|
||
username = urllib.parse.unquote(parsed_proxy.username)
|
||
password = urllib.parse.unquote(parsed_proxy.password or "")
|
||
encoded = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
|
||
return f"Basic {encoded}"
|
||
|
||
|
||
def _host_header(parsed: urllib.parse.SplitResult) -> str:
|
||
hostname = parsed.hostname or ""
|
||
try:
|
||
if ipaddress.ip_address(hostname).version == 6:
|
||
hostname = f"[{hostname}]"
|
||
except ValueError:
|
||
hostname = hostname.encode("idna").decode("ascii")
|
||
default = 443 if parsed.scheme == "https" else 80
|
||
return hostname if (parsed.port or default) == default else f"{hostname}:{parsed.port}"
|
||
|
||
|
||
def _build_ssl_context(options: ProbeOptions) -> ssl.SSLContext:
|
||
if options.insecure:
|
||
context = ssl._create_unverified_context() # noqa: SLF001 - explicit diagnostic flag
|
||
else:
|
||
context = ssl.create_default_context(cafile=options.ca_file)
|
||
with contextlib.suppress(NotImplementedError):
|
||
context.set_alpn_protocols(["http/1.1"])
|
||
return context
|
||
|
||
|
||
def _do_proxy_connect(
|
||
sock: socket.socket,
|
||
target_host: str,
|
||
target_port: int,
|
||
proxy_auth: str | None,
|
||
timeout: float,
|
||
) -> float:
|
||
try:
|
||
rendered_host = f"[{target_host}]" if ipaddress.ip_address(target_host).version == 6 else target_host
|
||
except ValueError:
|
||
rendered_host = target_host.encode("idna").decode("ascii")
|
||
authority = f"{rendered_host}:{target_port}"
|
||
lines = [
|
||
f"CONNECT {authority} HTTP/1.1",
|
||
f"Host: {authority}",
|
||
"Proxy-Connection: keep-alive",
|
||
]
|
||
if proxy_auth:
|
||
lines.append(f"Proxy-Authorization: {proxy_auth}")
|
||
request = ("\r\n".join(lines) + "\r\n\r\n").encode("iso-8859-1")
|
||
started = time.perf_counter()
|
||
try:
|
||
sock.sendall(request)
|
||
head, _body, _first = _recv_header_block(sock, timeout)
|
||
status, _reason, _headers = _parse_header_block(head)
|
||
except ProbeFailure as exc:
|
||
raise ProbeFailure("proxy_connect", f"{exc.phase}: {exc}") from exc
|
||
except OSError as exc:
|
||
raise ProbeFailure("proxy_connect", _safe_error(exc)) from exc
|
||
if not 200 <= status < 300:
|
||
raise ProbeFailure("proxy_connect", f"HTTP proxy CONNECT returned {status}")
|
||
return monotonic_ms(started)
|
||
|
||
|
||
def _request_target(parsed: urllib.parse.SplitResult, use_http_proxy: bool) -> str:
|
||
safe_path = urllib.parse.quote(parsed.path or "/", safe="/%:@-._~!$&'()*+,;=")
|
||
safe_query = urllib.parse.quote(parsed.query, safe="=&%/:@-._~!$'()*+,;?")
|
||
path = urllib.parse.urlunsplit(("", "", safe_path, safe_query, ""))
|
||
if not use_http_proxy:
|
||
return path
|
||
netloc = _host_header(parsed)
|
||
return urllib.parse.urlunsplit((parsed.scheme, netloc, safe_path, safe_query, ""))
|
||
|
||
|
||
def _content_length(headers: Mapping[str, str], *, transfer_encoded: bool) -> int | None:
|
||
raw = headers.get("content-length")
|
||
if raw is None or transfer_encoded:
|
||
return None
|
||
values = [part.strip() for part in raw.split(",")]
|
||
if not values or any(not re.fullmatch(r"\d+", value) for value in values) or len(set(values)) != 1:
|
||
raise ProbeFailure("body_framing", "invalid or conflicting Content-Length response header")
|
||
return int(values[0])
|
||
|
||
|
||
def _next_chunk_size(reader: _SocketBodyReader) -> int:
|
||
raw = reader.read_line()
|
||
size_text = raw.split(b";", 1)[0].strip()
|
||
if not size_text or not re.fullmatch(rb"[0-9A-Fa-f]+", size_text):
|
||
raise ProbeFailure("body_framing", "invalid HTTP chunk size")
|
||
size = int(size_text, 16)
|
||
if size > 2**63 - 1:
|
||
raise ProbeFailure("body_framing", "HTTP chunk size is unreasonably large")
|
||
return size
|
||
|
||
|
||
def _finish_chunk_trailers(reader: _SocketBodyReader) -> None:
|
||
total = 0
|
||
for _index in range(100):
|
||
line = reader.read_line()
|
||
total += len(line) + 2
|
||
if total > 65536:
|
||
raise ProbeFailure("body_framing", "HTTP chunk trailers exceeded 64 KiB")
|
||
if not line:
|
||
return
|
||
raise ProbeFailure("body_framing", "HTTP chunk trailers exceeded 100 fields")
|
||
|
||
|
||
def _read_response_body(
|
||
sock: socket.socket,
|
||
initial: bytes,
|
||
headers: Mapping[str, str],
|
||
*,
|
||
method_has_no_body: bool,
|
||
read_mode: str,
|
||
max_response_bytes: int,
|
||
idle_timeout: float,
|
||
first_body_timeout: float,
|
||
deadline: float,
|
||
) -> dict[str, Any]:
|
||
"""Read a response entity with Content-Length/chunked framing and hard bounds."""
|
||
transfer_tokens = [item.strip().lower() for item in headers.get("transfer-encoding", "").split(",") if item.strip()]
|
||
chunked = bool(transfer_tokens and transfer_tokens[-1] == "chunked")
|
||
content_length = _content_length(headers, transfer_encoded=bool(transfer_tokens))
|
||
waiting_for_first_entity_byte = read_mode == "first-byte"
|
||
reader = _SocketBodyReader(
|
||
sock,
|
||
initial,
|
||
idle_timeout=first_body_timeout if waiting_for_first_entity_byte else idle_timeout,
|
||
deadline=deadline,
|
||
idle_timeout_phase="first_body" if waiting_for_first_entity_byte else "body_idle_timeout",
|
||
)
|
||
result: dict[str, Any] = {
|
||
"body_bytes_read": 0,
|
||
"body_complete": False,
|
||
"body_truncated_reason": None,
|
||
"first_body_at": None,
|
||
"error_phase": None,
|
||
"error": None,
|
||
}
|
||
if transfer_tokens and not chunked:
|
||
result["framing_note"] = "non_chunked_transfer_encoding_delimited_by_connection_close"
|
||
elif transfer_tokens and headers.get("content-length") is not None:
|
||
result["framing_note"] = "transfer_encoding_overrides_content_length"
|
||
|
||
if method_has_no_body or content_length == 0:
|
||
result["body_complete"] = True
|
||
return result
|
||
if read_mode == "headers":
|
||
result["body_truncated_reason"] = "headers_only"
|
||
return result
|
||
|
||
try:
|
||
if chunked:
|
||
if read_mode == "first-byte":
|
||
while True:
|
||
size = _next_chunk_size(reader)
|
||
if size == 0:
|
||
_finish_chunk_trailers(reader)
|
||
result["body_complete"] = True
|
||
return result
|
||
first = reader.read_exact(1)
|
||
if first:
|
||
result["body_bytes_read"] = 1
|
||
result["first_body_at"] = time.perf_counter()
|
||
result["body_truncated_reason"] = "first_byte_mode"
|
||
return result
|
||
|
||
while True:
|
||
size = _next_chunk_size(reader)
|
||
if size == 0:
|
||
_finish_chunk_trailers(reader)
|
||
result["body_complete"] = True
|
||
return result
|
||
remaining_chunk = size
|
||
while remaining_chunk:
|
||
remaining_cap = max_response_bytes - int(result["body_bytes_read"])
|
||
if remaining_cap <= 0:
|
||
result["body_truncated_reason"] = "max_response_bytes"
|
||
return result
|
||
part = reader.read_some(min(65536, remaining_chunk, remaining_cap))
|
||
if not part:
|
||
raise ProbeFailure("body_framing", "response ended inside an HTTP chunk")
|
||
if result["first_body_at"] is None:
|
||
result["first_body_at"] = time.perf_counter()
|
||
result["body_bytes_read"] += len(part)
|
||
remaining_chunk -= len(part)
|
||
if reader.read_exact(2) != b"\r\n":
|
||
raise ProbeFailure("body_framing", "HTTP chunk data was not followed by CRLF")
|
||
if int(result["body_bytes_read"]) >= max_response_bytes:
|
||
# Do not wait for another chunk-size line once the configured
|
||
# entity-byte cap is reached. A streaming server is allowed
|
||
# to pause forever here; treating that pause as an idle
|
||
# timeout would falsely turn an intentional truncation into a
|
||
# transport failure. We can still recognize the common
|
||
# already-buffered empty-trailer terminator without blocking.
|
||
if reader.buffer.startswith(b"0\r\n\r\n"):
|
||
del reader.buffer[:5]
|
||
result["body_complete"] = True
|
||
else:
|
||
result["body_truncated_reason"] = "max_response_bytes"
|
||
return result
|
||
result["body_truncated_reason"] = "max_response_bytes"
|
||
return result
|
||
|
||
if read_mode == "first-byte":
|
||
first = reader.read_some(1)
|
||
if first:
|
||
result["body_bytes_read"] = 1
|
||
result["first_body_at"] = time.perf_counter()
|
||
if content_length == 1:
|
||
result["body_complete"] = True
|
||
else:
|
||
result["body_truncated_reason"] = "first_byte_mode"
|
||
elif content_length is None:
|
||
result["body_complete"] = True
|
||
else:
|
||
raise ProbeFailure("body_framing", "response ended before Content-Length bytes arrived")
|
||
return result
|
||
|
||
expected = content_length
|
||
while result["body_bytes_read"] < max_response_bytes:
|
||
if expected is not None:
|
||
remaining = expected - int(result["body_bytes_read"])
|
||
if remaining <= 0:
|
||
result["body_complete"] = True
|
||
return result
|
||
part = reader.read_some(min(65536, remaining, max_response_bytes - int(result["body_bytes_read"])))
|
||
if not part:
|
||
raise ProbeFailure("body_framing", "response ended before Content-Length bytes arrived")
|
||
else:
|
||
part = reader.read_some(min(65536, max_response_bytes - int(result["body_bytes_read"])))
|
||
if not part:
|
||
result["body_complete"] = True
|
||
return result
|
||
if result["first_body_at"] is None:
|
||
result["first_body_at"] = time.perf_counter()
|
||
result["body_bytes_read"] += len(part)
|
||
if expected is not None and result["body_bytes_read"] >= expected:
|
||
result["body_complete"] = True
|
||
else:
|
||
result["body_truncated_reason"] = "max_response_bytes"
|
||
except ProbeFailure as exc:
|
||
result["body_truncated_reason"] = exc.phase
|
||
result["error_phase"] = exc.phase
|
||
result["error"] = str(exc)[:300]
|
||
return result
|
||
|
||
|
||
def _probe_hop(url: str, options: ProbeOptions) -> tuple[dict[str, Any], str | None]:
|
||
hop_started = time.perf_counter()
|
||
hop_deadline = hop_started + options.total_timeout
|
||
parsed = urllib.parse.urlsplit(url)
|
||
scheme, target_host, target_port = _origin_tuple(url)
|
||
proxy_url: str | None = None
|
||
proxy_parsed: urllib.parse.SplitResult | None = None
|
||
connect_host, connect_port = target_host, target_port
|
||
proxy_auth: str | None = None
|
||
|
||
record: dict[str, Any] = {
|
||
"url": sanitize_url(url),
|
||
"method": options.method,
|
||
"proxy_used": False,
|
||
"proxy_origin": None,
|
||
"success": False,
|
||
"status_code": None,
|
||
"error_phase": None,
|
||
"error": None,
|
||
"timings_ms": {},
|
||
"response": {},
|
||
}
|
||
sock: socket.socket | ssl.SSLSocket | None = None
|
||
location: str | None = None
|
||
try:
|
||
proxy_url = _select_proxy(url, options)
|
||
if proxy_url:
|
||
proxy_parsed = urllib.parse.urlsplit(proxy_url)
|
||
if proxy_parsed.scheme.lower() != "http" or not proxy_parsed.hostname:
|
||
raise ProbeFailure("proxy", "only an http:// forward proxy is supported")
|
||
connect_host = proxy_parsed.hostname
|
||
connect_port = proxy_parsed.port or 80
|
||
proxy_auth = _proxy_authorization(proxy_parsed)
|
||
if proxy_auth is None:
|
||
proxy_auth = next(
|
||
(value for name, value in options.headers.items() if name.lower() == "proxy-authorization"),
|
||
None,
|
||
)
|
||
record["proxy_used"] = True
|
||
record["proxy_origin"] = sanitize_origin(proxy_url)
|
||
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("total_timeout", "overall request timeout expired before DNS")
|
||
sock, connect_details = _connect_socket(
|
||
connect_host,
|
||
connect_port,
|
||
timeout=min(options.connect_timeout, remaining),
|
||
family=options.ip_family,
|
||
)
|
||
record.update(connect_details)
|
||
record["dns_subject"] = "proxy" if proxy_parsed else "origin"
|
||
timings = record["timings_ms"]
|
||
timings["dns"] = connect_details["dns_ms"]
|
||
timings["tcp_connect"] = connect_details["connect_ms"]
|
||
|
||
if proxy_parsed and scheme == "https":
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("total_timeout", "overall request timeout expired before proxy CONNECT")
|
||
timings["proxy_connect"] = _do_proxy_connect(
|
||
sock,
|
||
target_host,
|
||
target_port,
|
||
proxy_auth,
|
||
min(options.first_byte_timeout, remaining),
|
||
)
|
||
|
||
if scheme == "https":
|
||
tls_started = time.perf_counter()
|
||
try:
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("total_timeout", "overall request timeout expired before TLS")
|
||
sock.settimeout(min(options.connect_timeout, remaining))
|
||
sock = _build_ssl_context(options).wrap_socket(sock, server_hostname=target_host)
|
||
except ssl.SSLCertVerificationError as exc:
|
||
raise ProbeFailure("tls_verify", _safe_error(exc)) from exc
|
||
except (TimeoutError, ssl.SSLError, OSError) as exc:
|
||
raise ProbeFailure("tls_handshake", _safe_error(exc)) from exc
|
||
timings["tls_handshake"] = monotonic_ms(tls_started)
|
||
cipher = sock.cipher()
|
||
cert = sock.getpeercert() or {}
|
||
record["tls"] = {
|
||
"verified": not options.insecure,
|
||
"version": sock.version(),
|
||
"cipher": cipher[0] if cipher else None,
|
||
"alpn": sock.selected_alpn_protocol(),
|
||
"certificate_not_after": cert.get("notAfter"),
|
||
}
|
||
|
||
request_headers = {name: value for name, value in options.headers.items() if name.lower() != "proxy-authorization"}
|
||
lower_names: dict[str, str] = {}
|
||
for name in request_headers:
|
||
lowered = name.lower()
|
||
if lowered in lower_names:
|
||
raise ProbeFailure(
|
||
"request_validation",
|
||
f"duplicate request header names differ only by case: {lower_names[lowered]} and {name}",
|
||
)
|
||
lower_names[lowered] = name
|
||
for forbidden in ("transfer-encoding", "expect"):
|
||
if forbidden in lower_names:
|
||
raise ProbeFailure("request_validation", f"request header {forbidden} is not supported")
|
||
supplied_length: int | None = None
|
||
if "content-length" in lower_names:
|
||
raw_length = request_headers.pop(lower_names["content-length"])
|
||
if not re.fullmatch(r"\d+", raw_length.strip()):
|
||
raise ProbeFailure("request_validation", "Content-Length must be a non-negative integer")
|
||
supplied_length = int(raw_length)
|
||
if supplied_length != len(options.body):
|
||
raise ProbeFailure("request_validation", "Content-Length does not match the request body")
|
||
if "connection" in lower_names:
|
||
request_headers.pop(lower_names["connection"])
|
||
defaults = {
|
||
"Host": _host_header(parsed),
|
||
"User-Agent": USER_AGENT,
|
||
"Accept": "*/*",
|
||
"Accept-Encoding": "identity",
|
||
"Connection": "close",
|
||
}
|
||
lower_names = {name.lower(): name for name in request_headers}
|
||
for name, value in defaults.items():
|
||
if name.lower() not in lower_names:
|
||
request_headers[name] = value
|
||
if options.body or supplied_length is not None or options.method not in SAFE_METHODS:
|
||
request_headers["Content-Length"] = str(len(options.body))
|
||
if proxy_parsed and scheme == "http" and proxy_auth:
|
||
request_headers["Proxy-Authorization"] = proxy_auth
|
||
for name, value in request_headers.items():
|
||
try:
|
||
_validate_header(name, value)
|
||
except ValueError as exc:
|
||
raise ProbeFailure("request_validation", str(exc)) from exc
|
||
|
||
target = _request_target(parsed, bool(proxy_parsed and scheme == "http"))
|
||
request_lines = [f"{options.method} {target} HTTP/1.1"]
|
||
request_lines.extend(f"{name}: {value}" for name, value in request_headers.items())
|
||
header_bytes = ("\r\n".join(request_lines) + "\r\n\r\n").encode("iso-8859-1")
|
||
if len(header_bytes) > MAX_REQUEST_HEADER_BYTES:
|
||
raise ProbeFailure("request_validation", "request headers exceed 64 KiB")
|
||
request_bytes = header_bytes + options.body
|
||
|
||
upload_started = time.perf_counter()
|
||
try:
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("total_timeout", "overall request timeout expired before upload")
|
||
sock.settimeout(min(options.connect_timeout, remaining))
|
||
sock.sendall(request_bytes)
|
||
except (TimeoutError, OSError) as exc:
|
||
raise ProbeFailure("request_upload", _safe_error(exc)) from exc
|
||
uploaded_at = time.perf_counter()
|
||
timings["request_upload"] = monotonic_ms(upload_started, uploaded_at)
|
||
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("total_timeout", "overall request timeout expired before response")
|
||
head, initial_body, first_response_at = _recv_header_block(
|
||
sock,
|
||
min(options.first_byte_timeout, remaining),
|
||
)
|
||
status, _reason, response_headers = _parse_header_block(head)
|
||
interim_statuses: list[int] = []
|
||
while 100 <= status < 200 and status != 101:
|
||
interim_statuses.append(status)
|
||
remaining = hop_deadline - time.perf_counter()
|
||
if remaining <= 0:
|
||
raise ProbeFailure("first_byte", "overall request timeout expired after interim response")
|
||
head, initial_body, _interim_first = _recv_header_block(
|
||
sock,
|
||
min(options.first_byte_timeout, remaining),
|
||
initial=initial_body,
|
||
)
|
||
status, _reason, response_headers = _parse_header_block(head)
|
||
headers_complete_at = time.perf_counter()
|
||
timings["response_wait"] = monotonic_ms(uploaded_at, first_response_at)
|
||
timings["headers_complete_after_upload"] = monotonic_ms(uploaded_at, headers_complete_at)
|
||
|
||
record["status_code"] = status
|
||
if interim_statuses:
|
||
record["interim_status_codes"] = interim_statuses
|
||
record["response"] = {
|
||
"headers": _safe_response_headers(response_headers),
|
||
"server_timing": parse_server_timing(response_headers.get("server-timing")),
|
||
"body_bytes_read": 0,
|
||
"body_complete": False,
|
||
"body_truncated_reason": None,
|
||
}
|
||
location = response_headers.get("location")
|
||
if location:
|
||
record["response"]["redirect_location"] = sanitize_url(urllib.parse.urljoin(url, location))
|
||
|
||
method_has_no_body = options.method == "HEAD" or status in {204, 304} or 100 <= status < 200
|
||
content_type = response_headers.get("content-type", "").lower()
|
||
read_mode = options.read_mode
|
||
if read_mode == "auto":
|
||
read_mode = "first-byte" if "text/event-stream" in content_type else "full"
|
||
|
||
body_started = time.perf_counter()
|
||
body_result = _read_response_body(
|
||
sock,
|
||
initial_body,
|
||
response_headers,
|
||
method_has_no_body=method_has_no_body,
|
||
read_mode=read_mode,
|
||
max_response_bytes=options.max_response_bytes,
|
||
idle_timeout=options.read_timeout,
|
||
first_body_timeout=options.first_byte_timeout,
|
||
deadline=hop_deadline,
|
||
)
|
||
body_finished_at = time.perf_counter()
|
||
first_body_at = body_result.pop("first_body_at")
|
||
body_error_phase = body_result.pop("error_phase")
|
||
body_error = body_result.pop("error")
|
||
record["response"].update(body_result)
|
||
if first_body_at is not None:
|
||
timings["first_body_after_upload"] = monotonic_ms(uploaded_at, first_body_at)
|
||
else:
|
||
timings["first_body_after_upload"] = None
|
||
timings["body_read"] = monotonic_ms(body_started, body_finished_at)
|
||
timings["total"] = monotonic_ms(hop_started, body_finished_at)
|
||
if body_error_phase:
|
||
record["error_phase"] = body_error_phase
|
||
record["error"] = body_error
|
||
else:
|
||
record["success"] = True
|
||
except ProbeFailure as exc:
|
||
record["error_phase"] = exc.phase
|
||
record["error"] = str(exc)[:300]
|
||
record["timings_ms"]["total"] = monotonic_ms(hop_started)
|
||
except Exception as exc: # pragma: no cover - last-resort containment
|
||
record["error_phase"] = "unexpected"
|
||
record["error"] = _safe_error(exc)
|
||
record["timings_ms"]["total"] = monotonic_ms(hop_started)
|
||
finally:
|
||
if sock is not None:
|
||
with contextlib.suppress(OSError):
|
||
sock.close()
|
||
return record, location
|
||
|
||
|
||
def probe_url(url: str, options: ProbeOptions) -> dict[str, Any]:
|
||
"""Probe one logical request, following redirects only when requested."""
|
||
started = time.perf_counter()
|
||
current_url = url
|
||
current_method = options.method
|
||
current_body = options.body
|
||
current_headers = dict(options.headers)
|
||
hops: list[dict[str, Any]] = []
|
||
visited: set[str] = set()
|
||
|
||
for hop_index in range(options.max_redirects + 1):
|
||
remaining_total = options.total_timeout - (time.perf_counter() - started)
|
||
if remaining_total <= 0:
|
||
hops.append(
|
||
{
|
||
"url": sanitize_url(current_url),
|
||
"method": current_method,
|
||
"success": False,
|
||
"status_code": None,
|
||
"error_phase": "total_timeout",
|
||
"error": "overall logical request timeout expired",
|
||
"timings_ms": {"total": monotonic_ms(started)},
|
||
"response": {},
|
||
"hop": hop_index,
|
||
}
|
||
)
|
||
break
|
||
hop_options = ProbeOptions(
|
||
method=current_method,
|
||
headers=current_headers,
|
||
body=current_body,
|
||
connect_timeout=options.connect_timeout,
|
||
first_byte_timeout=options.first_byte_timeout,
|
||
read_timeout=options.read_timeout,
|
||
total_timeout=remaining_total,
|
||
max_response_bytes=options.max_response_bytes,
|
||
read_mode=options.read_mode,
|
||
insecure=options.insecure,
|
||
ca_file=options.ca_file,
|
||
follow_redirects=options.follow_redirects,
|
||
max_redirects=options.max_redirects,
|
||
proxy=options.proxy,
|
||
use_env_proxy=options.use_env_proxy,
|
||
ip_family=options.ip_family,
|
||
)
|
||
hop, raw_location = _probe_hop(current_url, hop_options)
|
||
hop["hop"] = hop_index
|
||
hops.append(hop)
|
||
status = hop.get("status_code")
|
||
if not hop.get("success") or not options.follow_redirects or status not in {301, 302, 303, 307, 308}:
|
||
break
|
||
if not raw_location:
|
||
break
|
||
next_url = urllib.parse.urljoin(current_url, raw_location)
|
||
try:
|
||
next_origin = _origin_tuple(next_url)
|
||
except ValueError as exc:
|
||
hop["success"] = False
|
||
hop["error_phase"] = "redirect"
|
||
hop["error"] = f"invalid redirect target: {_safe_error(exc)}"
|
||
break
|
||
if next_url in visited:
|
||
hop["success"] = False
|
||
hop["error_phase"] = "redirect"
|
||
hop["error"] = "redirect loop detected"
|
||
break
|
||
visited.add(current_url)
|
||
if hop_index >= options.max_redirects:
|
||
hop["success"] = False
|
||
hop["error_phase"] = "redirect"
|
||
hop["error"] = "maximum redirects exceeded"
|
||
break
|
||
if _origin_tuple(current_url) != next_origin:
|
||
hop["success"] = False
|
||
hop["error_phase"] = "redirect"
|
||
hop["error"] = "cross-origin redirect refused to prevent credential or request-body disclosure"
|
||
break
|
||
if status == 303 or (status in {301, 302} and current_method == "POST"):
|
||
current_method = "GET"
|
||
current_body = b""
|
||
current_headers = {name: value for name, value in current_headers.items() if name.lower() not in {"content-length", "content-type"}}
|
||
current_url = next_url
|
||
|
||
final = hops[-1]
|
||
aggregate_timings: dict[str, float | None] = {}
|
||
for key in {
|
||
"dns",
|
||
"tcp_connect",
|
||
"proxy_connect",
|
||
"tls_handshake",
|
||
"request_upload",
|
||
"response_wait",
|
||
"headers_complete_after_upload",
|
||
"body_read",
|
||
}:
|
||
values = [hop.get("timings_ms", {}).get(key) for hop in hops]
|
||
clean = [float(value) for value in values if value is not None]
|
||
aggregate_timings[key] = round(sum(clean), 3) if clean else None
|
||
aggregate_timings["total"] = monotonic_ms(started)
|
||
return {
|
||
"url": sanitize_url(url),
|
||
"success": bool(final.get("success")),
|
||
"status_code": final.get("status_code"),
|
||
"error_phase": final.get("error_phase"),
|
||
"error": final.get("error"),
|
||
"timings_ms": aggregate_timings,
|
||
"hops": hops,
|
||
}
|
||
|
||
|
||
def summarize_probe_results(results: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
successful = [result for result in results if result.get("success")]
|
||
status_counts = Counter(str(result.get("status_code")) for result in successful)
|
||
error_counts = Counter(str(result.get("error_phase") or "unknown") for result in results if not result.get("success"))
|
||
timing_keys = [
|
||
"dns",
|
||
"tcp_connect",
|
||
"proxy_connect",
|
||
"tls_handshake",
|
||
"request_upload",
|
||
"response_wait",
|
||
"headers_complete_after_upload",
|
||
"body_read",
|
||
"total",
|
||
]
|
||
timings = {key: summarize_numbers(result.get("timings_ms", {}).get(key) for result in successful) for key in timing_keys}
|
||
return {
|
||
"requests": len(results),
|
||
"successful_transport": len(successful),
|
||
"transport_errors": len(results) - len(successful),
|
||
"status_counts": dict(status_counts),
|
||
"error_phase_counts": dict(error_counts),
|
||
"timings_ms": timings,
|
||
}
|
||
|
||
|
||
def run_probe_batch(
|
||
url: str,
|
||
options: ProbeOptions,
|
||
*,
|
||
count: int,
|
||
concurrency: int,
|
||
interval: float = 0.0,
|
||
) -> tuple[list[dict[str, Any]], float]:
|
||
"""Run a bounded number of fresh-connection probes."""
|
||
count = max(1, int(count))
|
||
concurrency = max(1, min(int(concurrency), count))
|
||
started = time.perf_counter()
|
||
results: list[dict[str, Any]] = []
|
||
if concurrency == 1:
|
||
for index in range(count):
|
||
results.append(probe_url(url, options))
|
||
if interval > 0 and index + 1 < count:
|
||
time.sleep(interval)
|
||
else:
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||
futures = [executor.submit(probe_url, url, options) for _ in range(count)]
|
||
for future in concurrent.futures.as_completed(futures):
|
||
results.append(future.result())
|
||
return results, monotonic_ms(started)
|
||
|
||
|
||
def validate_replay_safety(
|
||
*,
|
||
method: str,
|
||
url_count: int,
|
||
count: int,
|
||
concurrency: int,
|
||
follow_redirects: bool,
|
||
allow_unsafe_replay: bool,
|
||
) -> None:
|
||
method = method.upper()
|
||
if method in SAFE_METHODS:
|
||
return
|
||
total = max(1, url_count) * max(1, count)
|
||
replayed = total > 1 or concurrency > 1 or follow_redirects
|
||
if replayed and not allow_unsafe_replay:
|
||
raise ValueError(
|
||
"Unsafe method would be replayed. Login can invalidate an old session and failed login "
|
||
"can trigger IP lockout; agent POSTs can create duplicate runs. Use exactly one URL, "
|
||
"--count 1, --concurrency 1 and no redirect following, or explicitly add "
|
||
"--allow-unsafe-replay after reviewing the impact."
|
||
)
|
||
|
||
|
||
def _parse_scalar(value: str) -> Any:
|
||
raw = value.strip().strip("\"'")
|
||
lowered = raw.lower()
|
||
if lowered in {"true", "yes", "on"}:
|
||
return True
|
||
if lowered in {"false", "no", "off"}:
|
||
return False
|
||
if lowered in {"null", "none", "~", ""}:
|
||
return None
|
||
try:
|
||
return int(raw)
|
||
except ValueError:
|
||
try:
|
||
return float(raw)
|
||
except ValueError:
|
||
return raw
|
||
|
||
|
||
def read_safe_config(path: Path) -> dict[str, Any]:
|
||
"""Read a small allowlist of non-secret scalar YAML settings."""
|
||
wanted = {
|
||
"gateway.port",
|
||
"database.backend",
|
||
"database.pool_size",
|
||
"database.max_overflow",
|
||
"database.pool_timeout",
|
||
"database.pool_recycle",
|
||
"database.sqlite_dir",
|
||
"runtime_performance.skill_view_reuse_enabled",
|
||
"runtime_performance.skill_view_cache_enabled",
|
||
"runtime_performance.model_timing_nonblocking_enabled",
|
||
"runtime_performance.asyncio_thread_pool_enabled",
|
||
"runtime_performance.asyncio_thread_pool_size",
|
||
"concurrency.redis.enabled",
|
||
"concurrency.redis.socket_connect_timeout_ms",
|
||
"concurrency.redis.socket_timeout_ms",
|
||
"concurrency.redis.acquire_timeout_ms",
|
||
"concurrency.redis.fail_open",
|
||
}
|
||
output: dict[str, Any] = {}
|
||
stack: list[tuple[int, str]] = []
|
||
try:
|
||
lines = path.read_text(encoding="utf-8").splitlines()
|
||
except OSError as exc:
|
||
return {"path": str(path), "error": _safe_error(exc)}
|
||
for raw_line in lines:
|
||
line = raw_line.split("#", 1)[0].rstrip()
|
||
if not line.strip() or line.lstrip().startswith("-") or ":" not in line:
|
||
continue
|
||
indent = len(line) - len(line.lstrip(" \t"))
|
||
key, _separator, value = line.strip().partition(":")
|
||
while stack and stack[-1][0] >= indent:
|
||
stack.pop()
|
||
dotted = ".".join([item[1] for item in stack] + [key])
|
||
if value.strip():
|
||
if dotted in wanted:
|
||
output[dotted] = _parse_scalar(value)
|
||
else:
|
||
stack.append((indent, key))
|
||
output["path"] = str(path)
|
||
return output
|
||
|
||
|
||
def find_config_path(explicit: str | None = None) -> Path | None:
|
||
candidates: list[Path] = []
|
||
if explicit:
|
||
candidates.append(Path(explicit))
|
||
for env_name in ("DEER_FLOW_CONFIG_PATH", "DEER_FLOW_CONFIG", "DEERFLOW_CONFIG"):
|
||
if os.environ.get(env_name):
|
||
candidates.append(Path(os.environ[env_name]))
|
||
script_backend = Path(__file__).resolve().parent.parent
|
||
candidates.extend(
|
||
[
|
||
Path.cwd() / "config.yaml",
|
||
script_backend / "config.yaml",
|
||
Path("/app/backend/config.yaml"),
|
||
]
|
||
)
|
||
seen: set[Path] = set()
|
||
for candidate in candidates:
|
||
with contextlib.suppress(OSError):
|
||
resolved = candidate.expanduser().resolve()
|
||
if resolved in seen:
|
||
continue
|
||
seen.add(resolved)
|
||
if resolved.is_file():
|
||
return resolved
|
||
return None
|
||
|
||
|
||
def _runtime_home_candidates(config_path: Path | None) -> list[Path]:
|
||
candidates: list[Path] = []
|
||
if os.environ.get("DEER_FLOW_HOME"):
|
||
candidates.append(Path(os.environ["DEER_FLOW_HOME"]))
|
||
if config_path:
|
||
candidates.append(config_path.parent / ".deer-flow")
|
||
candidates.extend([Path.cwd() / ".deer-flow", Path("/app/backend/.deer-flow")])
|
||
return candidates
|
||
|
||
|
||
def read_safe_system_settings(config_path: Path | None) -> dict[str, Any]:
|
||
path = next(
|
||
(candidate / "system_settings.json" for candidate in _runtime_home_candidates(config_path) if (candidate / "system_settings.json").is_file()),
|
||
None,
|
||
)
|
||
if path is None:
|
||
return {"found": False}
|
||
try:
|
||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError) as exc:
|
||
return {"found": True, "path": str(path), "error": _safe_error(exc)}
|
||
model = raw.get("model_concurrency") if isinstance(raw, dict) else {}
|
||
monitor = raw.get("concurrency_monitor") if isinstance(raw, dict) else {}
|
||
if not isinstance(model, dict):
|
||
model = {}
|
||
if not isinstance(monitor, dict):
|
||
monitor = {}
|
||
safe_limits = {str(name)[:120]: int(value) for name, value in (model.get("model_limits") or {}).items() if isinstance(value, int) and value > 0}
|
||
safe_user_limits = {str(name)[:120]: int(value) for name, value in (model.get("user_model_limits") or {}).items() if isinstance(value, int) and value > 0}
|
||
return {
|
||
"found": True,
|
||
"path": str(path),
|
||
"concurrency_monitor_enabled": bool(monitor.get("enabled", False)),
|
||
"model_concurrency": {
|
||
"enabled": bool(model.get("enabled", False)),
|
||
"default_limit": model.get("default_limit"),
|
||
"user_default_limit": model.get("user_default_limit"),
|
||
"model_limits": safe_limits,
|
||
"user_model_limits": safe_user_limits,
|
||
},
|
||
}
|
||
|
||
|
||
def _run_command(
|
||
args: Sequence[str],
|
||
*,
|
||
timeout: float = 8.0,
|
||
env: Mapping[str, str] | None = None,
|
||
) -> dict[str, Any]:
|
||
started = time.perf_counter()
|
||
try:
|
||
completed = subprocess.run(
|
||
list(args),
|
||
check=False,
|
||
capture_output=True,
|
||
text=False,
|
||
timeout=timeout,
|
||
env=dict(env) if env is not None else None,
|
||
)
|
||
except FileNotFoundError:
|
||
return {"available": False, "returncode": None, "elapsed_ms": monotonic_ms(started), "output": b""}
|
||
except subprocess.TimeoutExpired as exc:
|
||
output = (exc.stdout or b"") + (exc.stderr or b"")
|
||
return {
|
||
"available": True,
|
||
"returncode": None,
|
||
"timed_out": True,
|
||
"elapsed_ms": monotonic_ms(started),
|
||
"output": output[:MAX_COMMAND_OUTPUT],
|
||
}
|
||
return {
|
||
"available": True,
|
||
"returncode": completed.returncode,
|
||
"timed_out": False,
|
||
"elapsed_ms": monotonic_ms(started),
|
||
"output": (completed.stdout + completed.stderr)[:MAX_COMMAND_OUTPUT],
|
||
}
|
||
|
||
|
||
def _read_proc_stat() -> dict[str, int]:
|
||
try:
|
||
first = Path("/proc/stat").read_text(encoding="ascii").splitlines()[0].split()
|
||
except (OSError, IndexError):
|
||
return {}
|
||
if not first or first[0] != "cpu":
|
||
return {}
|
||
names = ["user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice"]
|
||
values: dict[str, int] = {}
|
||
for name, raw in zip(names, first[1:]):
|
||
with contextlib.suppress(ValueError):
|
||
values[name] = int(raw)
|
||
return values
|
||
|
||
|
||
def _cpu_delta(before: Mapping[str, int], after: Mapping[str, int]) -> dict[str, float | None]:
|
||
keys = set(before) | set(after)
|
||
deltas = {key: max(0, int(after.get(key, 0)) - int(before.get(key, 0))) for key in keys}
|
||
total = sum(deltas.values()) - deltas.get("guest", 0) - deltas.get("guest_nice", 0)
|
||
if total <= 0:
|
||
return {"busy_pct": None, "iowait_pct": None, "steal_pct": None}
|
||
idle = deltas.get("idle", 0)
|
||
return {
|
||
"busy_pct": round(100.0 * (total - idle) / total, 2),
|
||
"iowait_pct": round(100.0 * deltas.get("iowait", 0) / total, 2),
|
||
"steal_pct": round(100.0 * deltas.get("steal", 0) / total, 2),
|
||
}
|
||
|
||
|
||
def _read_meminfo() -> dict[str, int]:
|
||
values: dict[str, int] = {}
|
||
try:
|
||
lines = Path("/proc/meminfo").read_text(encoding="ascii").splitlines()
|
||
except OSError:
|
||
return values
|
||
for line in lines:
|
||
key, separator, rest = line.partition(":")
|
||
if not separator:
|
||
continue
|
||
match = re.search(r"\d+", rest)
|
||
if match:
|
||
values[key] = int(match.group()) * 1024
|
||
return values
|
||
|
||
|
||
def _read_vmstat() -> dict[str, int]:
|
||
output: dict[str, int] = {}
|
||
try:
|
||
lines = Path("/proc/vmstat").read_text(encoding="ascii").splitlines()
|
||
except OSError:
|
||
return output
|
||
wanted = {"pswpin", "pswpout", "pgmajfault", "oom_kill"}
|
||
for line in lines:
|
||
key, _separator, raw = line.partition(" ")
|
||
if key not in wanted:
|
||
continue
|
||
with contextlib.suppress(ValueError):
|
||
output[key] = int(raw.strip())
|
||
return output
|
||
|
||
|
||
def _read_pressure() -> dict[str, Any]:
|
||
output: dict[str, Any] = {}
|
||
for resource in ("cpu", "memory", "io"):
|
||
path = Path("/proc/pressure") / resource
|
||
if not path.is_file():
|
||
continue
|
||
resource_values: dict[str, dict[str, float | int]] = {}
|
||
with contextlib.suppress(OSError):
|
||
for line in path.read_text(encoding="ascii").splitlines():
|
||
parts = line.split()
|
||
if not parts:
|
||
continue
|
||
metrics: dict[str, float | int] = {}
|
||
for part in parts[1:]:
|
||
key, separator, raw = part.partition("=")
|
||
if not separator:
|
||
continue
|
||
try:
|
||
metrics[key] = int(raw) if key == "total" else float(raw)
|
||
except ValueError:
|
||
pass
|
||
resource_values[parts[0]] = metrics
|
||
if resource_values:
|
||
output[resource] = resource_values
|
||
return output
|
||
|
||
|
||
def _read_diskstats() -> dict[str, dict[str, int]]:
|
||
output: dict[str, dict[str, int]] = {}
|
||
try:
|
||
lines = Path("/proc/diskstats").read_text(encoding="ascii").splitlines()
|
||
except OSError:
|
||
return output
|
||
for line in lines:
|
||
parts = line.split()
|
||
if len(parts) < 14:
|
||
continue
|
||
name = parts[2]
|
||
if not (Path("/sys/block") / name).exists():
|
||
continue
|
||
try:
|
||
output[name] = {
|
||
"reads": int(parts[3]),
|
||
"sectors_read": int(parts[5]),
|
||
"writes": int(parts[7]),
|
||
"sectors_written": int(parts[9]),
|
||
"io_ms": int(parts[12]),
|
||
}
|
||
except ValueError:
|
||
continue
|
||
return output
|
||
|
||
|
||
def _disk_delta(
|
||
before: Mapping[str, Mapping[str, int]],
|
||
after: Mapping[str, Mapping[str, int]],
|
||
elapsed_seconds: float,
|
||
) -> dict[str, Any]:
|
||
devices: list[dict[str, Any]] = []
|
||
for name in sorted(set(before) & set(after)):
|
||
old = before[name]
|
||
new = after[name]
|
||
read_bytes = max(0, new["sectors_read"] - old["sectors_read"]) * 512
|
||
write_bytes = max(0, new["sectors_written"] - old["sectors_written"]) * 512
|
||
io_ms = max(0, new["io_ms"] - old["io_ms"])
|
||
devices.append(
|
||
{
|
||
"device": name,
|
||
"read_bytes_per_sec": round(read_bytes / max(elapsed_seconds, 0.001), 1),
|
||
"write_bytes_per_sec": round(write_bytes / max(elapsed_seconds, 0.001), 1),
|
||
"util_pct": round(min(100.0, io_ms / (elapsed_seconds * 10.0)), 2),
|
||
"reads": max(0, new["reads"] - old["reads"]),
|
||
"writes": max(0, new["writes"] - old["writes"]),
|
||
}
|
||
)
|
||
devices.sort(key=lambda item: item["util_pct"], reverse=True)
|
||
return {
|
||
"devices": devices[:12],
|
||
"max_util_pct": devices[0]["util_pct"] if devices else None,
|
||
"busiest_device": devices[0]["device"] if devices else None,
|
||
}
|
||
|
||
|
||
def _read_netdev() -> dict[str, dict[str, int]]:
|
||
output: dict[str, dict[str, int]] = {}
|
||
try:
|
||
lines = Path("/proc/net/dev").read_text(encoding="ascii").splitlines()[2:]
|
||
except OSError:
|
||
return output
|
||
for line in lines:
|
||
name, separator, raw = line.partition(":")
|
||
if not separator:
|
||
continue
|
||
interface = name.strip()
|
||
fields = raw.split()
|
||
if len(fields) < 16:
|
||
continue
|
||
with contextlib.suppress(ValueError):
|
||
output[interface] = {
|
||
"rx_bytes": int(fields[0]),
|
||
"rx_errors": int(fields[2]),
|
||
"rx_dropped": int(fields[3]),
|
||
"tx_bytes": int(fields[8]),
|
||
"tx_errors": int(fields[10]),
|
||
"tx_dropped": int(fields[11]),
|
||
}
|
||
return output
|
||
|
||
|
||
def _net_delta(
|
||
before: Mapping[str, Mapping[str, int]],
|
||
after: Mapping[str, Mapping[str, int]],
|
||
elapsed_seconds: float,
|
||
) -> dict[str, Any]:
|
||
totals = Counter()
|
||
for name in set(before) & set(after):
|
||
if name == "lo":
|
||
continue
|
||
for key in before[name]:
|
||
totals[key] += max(0, int(after[name].get(key, 0)) - int(before[name].get(key, 0)))
|
||
return {
|
||
"rx_bytes_per_sec": round(totals["rx_bytes"] / max(elapsed_seconds, 0.001), 1),
|
||
"tx_bytes_per_sec": round(totals["tx_bytes"] / max(elapsed_seconds, 0.001), 1),
|
||
"rx_errors_delta": totals["rx_errors"],
|
||
"tx_errors_delta": totals["tx_errors"],
|
||
"rx_dropped_delta": totals["rx_dropped"],
|
||
"tx_dropped_delta": totals["tx_dropped"],
|
||
}
|
||
|
||
|
||
def _read_cgroup_number(path: str) -> int | None:
|
||
try:
|
||
raw = Path(path).read_text(encoding="ascii").strip()
|
||
except OSError:
|
||
return None
|
||
if raw == "max":
|
||
return None
|
||
with contextlib.suppress(ValueError):
|
||
return int(raw)
|
||
return None
|
||
|
||
|
||
def _read_cgroup_limits() -> dict[str, Any]:
|
||
output: dict[str, Any] = {}
|
||
if Path("/sys/fs/cgroup/cgroup.controllers").exists():
|
||
memory_current = _read_cgroup_number("/sys/fs/cgroup/memory.current")
|
||
memory_max = _read_cgroup_number("/sys/fs/cgroup/memory.max")
|
||
pids_current = _read_cgroup_number("/sys/fs/cgroup/pids.current")
|
||
pids_max = _read_cgroup_number("/sys/fs/cgroup/pids.max")
|
||
cpu_quota: float | None = None
|
||
with contextlib.suppress(OSError, ValueError):
|
||
quota_raw, period_raw = Path("/sys/fs/cgroup/cpu.max").read_text(encoding="ascii").split()[:2]
|
||
if quota_raw != "max":
|
||
cpu_quota = round(int(quota_raw) / int(period_raw), 3)
|
||
output = {
|
||
"version": 2,
|
||
"memory_current_bytes": memory_current,
|
||
"memory_limit_bytes": memory_max,
|
||
"memory_usage_pct": (round(100.0 * memory_current / memory_max, 2) if memory_current is not None and memory_max and memory_max > 0 else None),
|
||
"cpu_limit_cores": cpu_quota,
|
||
"pids_current": pids_current,
|
||
"pids_limit": pids_max,
|
||
}
|
||
return output
|
||
|
||
|
||
def _read_process_stat(pid: int) -> tuple[int, int] | None:
|
||
try:
|
||
raw = (Path("/proc") / str(pid) / "stat").read_text(encoding="ascii")
|
||
after_name = raw.rsplit(")", 1)[1].split()
|
||
return int(after_name[11]), int(after_name[12])
|
||
except (OSError, IndexError, ValueError):
|
||
return None
|
||
|
||
|
||
def _classify_process(cmdline: str, name: str) -> str | None:
|
||
lowered = f"{name} {cmdline}".lower()
|
||
if "nginx" in lowered:
|
||
return "nginx"
|
||
if any(token in lowered for token in ("mysqld", "mariadbd")):
|
||
return "mysql"
|
||
if any(token in lowered for token in ("uvicorn", "gunicorn", "deerflow", "app.gateway")):
|
||
return "deerflow_gateway"
|
||
return None
|
||
|
||
|
||
def _scan_relevant_processes() -> dict[int, dict[str, Any]]:
|
||
output: dict[int, dict[str, Any]] = {}
|
||
for path in Path("/proc").iterdir():
|
||
if not path.name.isdigit():
|
||
continue
|
||
pid = int(path.name)
|
||
try:
|
||
cmdline = (path / "cmdline").read_bytes().replace(b"\x00", b" ").decode(errors="replace")
|
||
status_lines = (path / "status").read_text(encoding="utf-8", errors="replace").splitlines()
|
||
except OSError:
|
||
continue
|
||
status: dict[str, str] = {}
|
||
for line in status_lines:
|
||
key, separator, value = line.partition(":")
|
||
if separator:
|
||
status[key] = value.strip()
|
||
role = _classify_process(cmdline, status.get("Name", ""))
|
||
if role is None:
|
||
continue
|
||
fd_count: int | None = None
|
||
with contextlib.suppress(OSError):
|
||
fd_count = len(list((path / "fd").iterdir()))
|
||
fd_limit: int | None = None
|
||
with contextlib.suppress(OSError):
|
||
for line in (path / "limits").read_text(encoding="utf-8").splitlines():
|
||
if line.startswith("Max open files"):
|
||
fields = line.split()
|
||
if len(fields) >= 4 and fields[3].isdigit():
|
||
fd_limit = int(fields[3])
|
||
rss_kib = 0
|
||
match = re.search(r"\d+", status.get("VmRSS", ""))
|
||
if match:
|
||
rss_kib = int(match.group())
|
||
output[pid] = {
|
||
"pid": pid,
|
||
"role": role,
|
||
"rss_bytes": rss_kib * 1024,
|
||
"threads": int(status.get("Threads", "0") or 0),
|
||
"fd_count": fd_count,
|
||
"fd_limit": fd_limit,
|
||
"cpu_ticks": _read_process_stat(pid),
|
||
}
|
||
return output
|
||
|
||
|
||
def _process_delta(
|
||
before: Mapping[int, Mapping[str, Any]],
|
||
after: Mapping[int, Mapping[str, Any]],
|
||
elapsed_seconds: float,
|
||
) -> dict[str, Any]:
|
||
tick_rate = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
|
||
grouped: dict[str, dict[str, Any]] = defaultdict(
|
||
lambda: {
|
||
"processes": 0,
|
||
"rss_bytes": 0,
|
||
"threads": 0,
|
||
"fd_count": 0,
|
||
"fd_limit_min": None,
|
||
"cpu_pct_sum": 0.0,
|
||
"max_fd_usage_pct": None,
|
||
}
|
||
)
|
||
for pid, item in after.items():
|
||
role = str(item["role"])
|
||
group = grouped[role]
|
||
group["processes"] += 1
|
||
group["rss_bytes"] += int(item.get("rss_bytes") or 0)
|
||
group["threads"] += int(item.get("threads") or 0)
|
||
fd_count = item.get("fd_count")
|
||
fd_limit = item.get("fd_limit")
|
||
if isinstance(fd_count, int):
|
||
group["fd_count"] += fd_count
|
||
if isinstance(fd_limit, int) and fd_limit > 0:
|
||
current_min = group["fd_limit_min"]
|
||
group["fd_limit_min"] = fd_limit if current_min is None else min(current_min, fd_limit)
|
||
if isinstance(fd_count, int):
|
||
usage = round(100.0 * fd_count / fd_limit, 2)
|
||
current_usage = group["max_fd_usage_pct"]
|
||
group["max_fd_usage_pct"] = usage if current_usage is None else max(current_usage, usage)
|
||
old = before.get(pid, {}).get("cpu_ticks")
|
||
new = item.get("cpu_ticks")
|
||
if old and new:
|
||
ticks = max(0, (new[0] + new[1]) - (old[0] + old[1]))
|
||
group["cpu_pct_sum"] += 100.0 * ticks / max(tick_rate * elapsed_seconds, 0.001)
|
||
for group in grouped.values():
|
||
group["cpu_pct_sum"] = round(group["cpu_pct_sum"], 2)
|
||
return dict(grouped)
|
||
|
||
|
||
def collect_linux_snapshot(sample_seconds: float = 1.0) -> dict[str, Any]:
|
||
if platform.system() != "Linux" or not Path("/proc").is_dir():
|
||
return {"supported": False, "reason": "server checks require Linux /proc"}
|
||
sample_seconds = max(0.25, min(float(sample_seconds), 5.0))
|
||
cpu_before = _read_proc_stat()
|
||
disk_before = _read_diskstats()
|
||
net_before = _read_netdev()
|
||
vm_before = _read_vmstat()
|
||
process_before = _scan_relevant_processes()
|
||
started = time.perf_counter()
|
||
time.sleep(sample_seconds)
|
||
elapsed = time.perf_counter() - started
|
||
cpu_after = _read_proc_stat()
|
||
disk_after = _read_diskstats()
|
||
net_after = _read_netdev()
|
||
vm_after = _read_vmstat()
|
||
process_after = _scan_relevant_processes()
|
||
|
||
mem = _read_meminfo()
|
||
total_memory = mem.get("MemTotal", 0)
|
||
available_memory = mem.get("MemAvailable", 0)
|
||
swap_total = mem.get("SwapTotal", 0)
|
||
swap_free = mem.get("SwapFree", 0)
|
||
load_values: tuple[float, float, float] | tuple[()] = ()
|
||
with contextlib.suppress(OSError):
|
||
load_values = os.getloadavg()
|
||
file_nr: list[int] = []
|
||
with contextlib.suppress(OSError, ValueError):
|
||
file_nr = [int(item) for item in Path("/proc/sys/fs/file-nr").read_text().split()]
|
||
file_max = _read_cgroup_number("/proc/sys/fs/file-max")
|
||
|
||
disk_paths: dict[str, Any] = {}
|
||
for path in (Path("/"), Path.cwd()):
|
||
try:
|
||
usage = shutil.disk_usage(path)
|
||
except OSError:
|
||
continue
|
||
disk_paths[str(path)] = {
|
||
"total_bytes": usage.total,
|
||
"free_bytes": usage.free,
|
||
"free_pct": round(100.0 * usage.free / usage.total, 2) if usage.total else None,
|
||
}
|
||
|
||
return {
|
||
"supported": True,
|
||
"sample_seconds": round(elapsed, 3),
|
||
"hostname_hash": hashlib.sha256(socket.gethostname().encode()).hexdigest()[:12],
|
||
"kernel": platform.release(),
|
||
"cpu_count": os.cpu_count(),
|
||
"load_average": list(load_values),
|
||
"cpu": _cpu_delta(cpu_before, cpu_after),
|
||
"memory": {
|
||
"total_bytes": total_memory,
|
||
"available_bytes": available_memory,
|
||
"available_pct": round(100.0 * available_memory / total_memory, 2) if total_memory else None,
|
||
"swap_total_bytes": swap_total,
|
||
"swap_used_bytes": max(0, swap_total - swap_free),
|
||
"swap_used_pct": round(100.0 * (swap_total - swap_free) / swap_total, 2) if swap_total else 0.0,
|
||
"dirty_bytes": mem.get("Dirty", 0),
|
||
},
|
||
"vm_delta": {key: max(0, vm_after.get(key, 0) - vm_before.get(key, 0)) for key in set(vm_before) | set(vm_after)},
|
||
"pressure": _read_pressure(),
|
||
"disk_io": _disk_delta(disk_before, disk_after, elapsed),
|
||
"disk_space": disk_paths,
|
||
"network": _net_delta(net_before, net_after, elapsed),
|
||
"processes": _process_delta(process_before, process_after, elapsed),
|
||
"cgroup": _read_cgroup_limits(),
|
||
"file_handles": {
|
||
"allocated": file_nr[0] if file_nr else None,
|
||
"unused": file_nr[1] if len(file_nr) > 1 else None,
|
||
"system_limit": file_max or (file_nr[2] if len(file_nr) > 2 else None),
|
||
},
|
||
}
|
||
|
||
|
||
def _port_from_address(value: str) -> int | None:
|
||
if value.startswith("[") and "]:" in value:
|
||
raw = value.rsplit(":", 1)[-1]
|
||
elif ":" in value:
|
||
raw = value.rsplit(":", 1)[-1]
|
||
else:
|
||
return None
|
||
with contextlib.suppress(ValueError):
|
||
return int(raw)
|
||
return None
|
||
|
||
|
||
def collect_socket_snapshot(relevant_ports: set[int]) -> dict[str, Any]:
|
||
if shutil.which("ss") is None:
|
||
return {"available": False, "reason": "ss command not found"}
|
||
listeners_result = _run_command(["ss", "-H", "-lnt"], timeout=5)
|
||
connections_result = _run_command(["ss", "-H", "-tan"], timeout=8)
|
||
listeners: list[dict[str, Any]] = []
|
||
if listeners_result.get("returncode") == 0:
|
||
for raw_line in listeners_result["output"].decode(errors="replace").splitlines():
|
||
parts = raw_line.split()
|
||
if len(parts) < 5:
|
||
continue
|
||
port = _port_from_address(parts[3])
|
||
if port is None or (relevant_ports and port not in relevant_ports):
|
||
continue
|
||
try:
|
||
receive_queue = int(parts[1])
|
||
backlog = int(parts[2])
|
||
except ValueError:
|
||
continue
|
||
listeners.append(
|
||
{
|
||
"port": port,
|
||
"receive_queue": receive_queue,
|
||
"backlog": backlog,
|
||
"backlog_usage_pct": round(100.0 * receive_queue / backlog, 2) if backlog else None,
|
||
}
|
||
)
|
||
global_states: Counter[str] = Counter()
|
||
by_port: dict[int, Counter[str]] = defaultdict(Counter)
|
||
if connections_result.get("returncode") == 0:
|
||
for raw_line in connections_result["output"].decode(errors="replace").splitlines():
|
||
parts = raw_line.split()
|
||
if len(parts) < 5:
|
||
continue
|
||
state = parts[0].upper()
|
||
global_states[state] += 1
|
||
local_port = _port_from_address(parts[3])
|
||
peer_port = _port_from_address(parts[4])
|
||
for port in relevant_ports:
|
||
if port in {local_port, peer_port}:
|
||
by_port[port][state] += 1
|
||
return {
|
||
"available": bool(listeners_result.get("available") and connections_result.get("available")),
|
||
"listeners": listeners,
|
||
"global_states": dict(global_states),
|
||
"states_by_relevant_port": {str(port): dict(states) for port, states in by_port.items()},
|
||
}
|
||
|
||
|
||
def _parse_nginx_config(text: str, *, source: str) -> dict[str, Any]:
|
||
def _last_value(pattern: str) -> str | None:
|
||
matches = re.findall(pattern, text, flags=re.IGNORECASE | re.MULTILINE)
|
||
return matches[-1] if matches else None
|
||
|
||
worker_processes_raw = _last_value(r"^\s*worker_processes\s+([^;\s]+)\s*;")
|
||
worker_connections_raw = _last_value(r"^\s*worker_connections\s+(\d+)\s*;")
|
||
worker_rlimit_raw = _last_value(r"^\s*worker_rlimit_nofile\s+(\d+)\s*;")
|
||
keepalive_timeout_raw = _last_value(r"^\s*keepalive_timeout\s+([^;]+)\s*;")
|
||
multi_accept_raw = _last_value(r"^\s*multi_accept\s+([^;\s]+)\s*;")
|
||
workers: int | None
|
||
if worker_processes_raw == "auto":
|
||
workers = os.cpu_count()
|
||
else:
|
||
try:
|
||
workers = int(worker_processes_raw) if worker_processes_raw else None
|
||
except ValueError:
|
||
workers = None
|
||
worker_connections = int(worker_connections_raw) if worker_connections_raw else 512
|
||
theoretical = workers * worker_connections if workers else None
|
||
return {
|
||
"source": source,
|
||
"worker_processes_config": worker_processes_raw,
|
||
"worker_processes_resolved": workers,
|
||
"worker_connections": worker_connections,
|
||
"worker_rlimit_nofile": int(worker_rlimit_raw) if worker_rlimit_raw else None,
|
||
"theoretical_connection_slots": theoretical,
|
||
"reverse_proxy_estimated_client_ceiling": theoretical // 2 if theoretical else None,
|
||
"multi_accept": multi_accept_raw,
|
||
"keepalive_timeout": keepalive_timeout_raw[:80] if keepalive_timeout_raw else None,
|
||
"limit_conn_directives": len(re.findall(r"\blimit_conn\s+", text)),
|
||
"upstream_keepalive_directives": len(re.findall(r"\bkeepalive\s+\d+\s*;", text)),
|
||
"proxy_connect_timeout": _last_value(r"^\s*proxy_connect_timeout\s+([^;]+)\s*;"),
|
||
"proxy_read_timeout": _last_value(r"^\s*proxy_read_timeout\s+([^;]+)\s*;"),
|
||
"proxy_send_timeout": _last_value(r"^\s*proxy_send_timeout\s+([^;]+)\s*;"),
|
||
}
|
||
|
||
|
||
def collect_nginx_config(container_ids: Sequence[str] = ()) -> dict[str, Any]:
|
||
candidates: list[dict[str, Any]] = []
|
||
if shutil.which("nginx"):
|
||
result = _run_command(["nginx", "-T"], timeout=8)
|
||
if result.get("returncode") == 0:
|
||
candidates.append(_parse_nginx_config(result["output"].decode(errors="replace"), source="host:nginx -T"))
|
||
if shutil.which("docker"):
|
||
for container_id in list(container_ids)[:5]:
|
||
result = _run_command(["docker", "exec", container_id, "nginx", "-T"], timeout=8)
|
||
if result.get("returncode") == 0:
|
||
candidates.append(
|
||
_parse_nginx_config(
|
||
result["output"].decode(errors="replace"),
|
||
source=f"container:{container_id[:12]}",
|
||
)
|
||
)
|
||
if not candidates:
|
||
return {"available": False, "reason": "no readable running Nginx configuration"}
|
||
return {"available": True, "instances": candidates}
|
||
|
||
|
||
def collect_docker_snapshot() -> dict[str, Any]:
|
||
if shutil.which("docker") is None:
|
||
return {"available": False, "reason": "docker command not found"}
|
||
listing = _run_command(
|
||
["docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}"],
|
||
timeout=8,
|
||
)
|
||
if listing.get("returncode") != 0:
|
||
return {"available": False, "reason": "docker ps failed"}
|
||
containers: list[dict[str, Any]] = []
|
||
relevant_ids: list[str] = []
|
||
nginx_ids: list[str] = []
|
||
for line in listing["output"].decode(errors="replace").splitlines():
|
||
parts = line.split("\t", 3)
|
||
if len(parts) != 4:
|
||
continue
|
||
container_id, name, image, status = parts
|
||
lowered = f"{name} {image}".lower()
|
||
if not any(token in lowered for token in ("deerflow", "gateway", "backend", "nginx", "frontend", "mysql", "mariadb")):
|
||
continue
|
||
role = "other"
|
||
if any(token in lowered for token in ("mysql", "mariadb")):
|
||
role = "mysql"
|
||
elif "nginx" in lowered or "frontend" in lowered:
|
||
role = "nginx_or_frontend"
|
||
nginx_ids.append(container_id)
|
||
elif any(token in lowered for token in ("deerflow", "gateway", "backend")):
|
||
role = "deerflow_gateway"
|
||
item = {
|
||
"id": container_id[:12],
|
||
"name": name[:120],
|
||
"image": image[:180],
|
||
"role": role,
|
||
"status": status[:160],
|
||
}
|
||
containers.append(item)
|
||
relevant_ids.append(container_id)
|
||
|
||
stats_by_name: dict[str, dict[str, Any]] = {}
|
||
if relevant_ids:
|
||
stats = _run_command(
|
||
[
|
||
"docker",
|
||
"stats",
|
||
"--no-stream",
|
||
"--format",
|
||
"{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}\t{{.NetIO}}\t{{.BlockIO}}",
|
||
*relevant_ids,
|
||
],
|
||
timeout=15,
|
||
)
|
||
if stats.get("returncode") == 0:
|
||
for line in stats["output"].decode(errors="replace").splitlines():
|
||
fields = line.split("\t")
|
||
if len(fields) != 7:
|
||
continue
|
||
name, cpu, memory, memory_pct, pids, network, block = fields
|
||
stats_by_name[name] = {
|
||
"cpu_pct": _parse_percent(cpu),
|
||
"memory_usage": memory[:100],
|
||
"memory_pct": _parse_percent(memory_pct),
|
||
"pids": int(pids) if pids.isdigit() else None,
|
||
"network_io": network[:100],
|
||
"block_io": block[:100],
|
||
}
|
||
for container, container_id in zip(containers, relevant_ids):
|
||
container["stats"] = stats_by_name.get(container["name"], {})
|
||
inspect = _run_command(
|
||
[
|
||
"docker",
|
||
"inspect",
|
||
"--format",
|
||
"{{.HostConfig.Memory}}\t{{.HostConfig.NanoCpus}}\t{{.HostConfig.PidsLimit}}\t{{.State.OOMKilled}}\t{{.RestartCount}}",
|
||
container_id,
|
||
],
|
||
timeout=5,
|
||
)
|
||
if inspect.get("returncode") == 0:
|
||
fields = inspect["output"].decode(errors="replace").strip().split("\t")
|
||
if len(fields) == 5:
|
||
memory_limit, nano_cpus, pids_limit, oom_killed, restart_count = fields
|
||
container["limits"] = {
|
||
"memory_bytes": int(memory_limit) if memory_limit.isdigit() and int(memory_limit) > 0 else None,
|
||
"cpu_cores": round(int(nano_cpus) / 1_000_000_000, 3) if nano_cpus.isdigit() and int(nano_cpus) > 0 else None,
|
||
"pids": int(pids_limit) if re.fullmatch(r"-?\d+", pids_limit) and int(pids_limit) > 0 else None,
|
||
"oom_killed": oom_killed.lower() == "true",
|
||
"restart_count": int(restart_count) if restart_count.isdigit() else None,
|
||
}
|
||
return {"available": True, "containers": containers, "nginx_container_ids": nginx_ids}
|
||
|
||
|
||
def _parse_percent(raw: str) -> float | None:
|
||
try:
|
||
return round(float(raw.strip().rstrip("%")), 2)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
LOG_PATTERNS: dict[str, re.Pattern[str]] = {
|
||
"nginx_connection_capacity": re.compile(r"worker_connections are not enough", re.IGNORECASE),
|
||
"too_many_open_files": re.compile(r"too many open files", re.IGNORECASE),
|
||
"nginx_upstream_timeout": re.compile(r"upstream timed out", re.IGNORECASE),
|
||
"nginx_upstream_connect_failure": re.compile(r"connect\(\) failed|no live upstreams|upstream.*connection refused", re.IGNORECASE),
|
||
"db_pool_timeout": re.compile(r"QueuePool limit|connection pool.*(?:exhaust|timeout)|pool timeout", re.IGNORECASE),
|
||
"db_lock_or_deadlock": re.compile(r"lock wait timeout|deadlock found|database is locked", re.IGNORECASE),
|
||
"db_disconnect": re.compile(r"server has gone away|lost connection.*mysql|connection.*closed|connection reset", re.IGNORECASE),
|
||
"deerflow_concurrency_reject": re.compile(r"reached the concurrency limit|selected model is busy|ModelConcurrencyLimit", re.IGNORECASE),
|
||
"resource_exhaustion": re.compile(r"cannot allocate memory|out of memory|oomkilled|resource temporarily unavailable", re.IGNORECASE),
|
||
"gateway_timeout": re.compile(r"504 Gateway Time-out|Gateway Timeout", re.IGNORECASE),
|
||
"skill_view_timing": re.compile(r"SKILL_VIEW_CACHE", re.IGNORECASE),
|
||
"start_run_timing": re.compile(r"\[start-run-timing\]", re.IGNORECASE),
|
||
"operation_timing": re.compile(r"GATEWAY_OPERATION_TIMING", re.IGNORECASE),
|
||
"llm_request_timing": re.compile(r"LLM_REQUEST_(?:DONE|FAILED)", re.IGNORECASE),
|
||
}
|
||
|
||
|
||
def _tail_bytes(path: Path, maximum: int) -> bytes:
|
||
with path.open("rb") as handle:
|
||
handle.seek(0, os.SEEK_END)
|
||
size = handle.tell()
|
||
handle.seek(max(0, size - maximum))
|
||
return handle.read(maximum)
|
||
|
||
|
||
def discover_log_paths(explicit: Sequence[str], config_path: Path | None) -> list[Path]:
|
||
candidates = [Path(path) for path in explicit]
|
||
roots = [Path.cwd(), Path(__file__).resolve().parents[2]]
|
||
if config_path:
|
||
roots.append(config_path.parent)
|
||
for root in roots:
|
||
candidates.extend(
|
||
[
|
||
root / ".runtime/logs/backend.log",
|
||
root / ".runtime/logs/gateway.log",
|
||
root / "logs/backend.log",
|
||
root / "logs/gateway.log",
|
||
]
|
||
)
|
||
for pattern in ("/var/log/nginx/*.log", "/var/log/mysql/*.log", "/var/log/mariadb/*.log"):
|
||
candidates.extend(Path(item) for item in glob.glob(pattern))
|
||
output: list[Path] = []
|
||
seen: set[Path] = set()
|
||
for candidate in candidates:
|
||
with contextlib.suppress(OSError):
|
||
resolved = candidate.expanduser().resolve()
|
||
if resolved not in seen and resolved.is_file():
|
||
seen.add(resolved)
|
||
output.append(resolved)
|
||
return output
|
||
|
||
|
||
def scan_logs(paths: Sequence[Path], tail_bytes: int = 4 * 1024 * 1024) -> dict[str, Any]:
|
||
files: list[dict[str, Any]] = []
|
||
totals: Counter[str] = Counter()
|
||
timing_max: dict[str, float] = {}
|
||
for path in paths:
|
||
try:
|
||
text = _tail_bytes(path, tail_bytes).decode("utf-8", errors="replace")
|
||
stat = path.stat()
|
||
except OSError as exc:
|
||
files.append({"path": str(path), "error": _safe_error(exc)})
|
||
continue
|
||
counts: dict[str, int] = {}
|
||
for name, pattern in LOG_PATTERNS.items():
|
||
count = len(pattern.findall(text))
|
||
if count:
|
||
counts[name] = count
|
||
totals[name] += count
|
||
numeric_patterns = {
|
||
"skill_view_total_ms": r"SKILL_VIEW_CACHE[^\n]*\btotal_ms=([0-9.]+)",
|
||
"agent_build_elapsed_ms": r"AGENT_BUILD_TIMING[^\n]*\belapsed_ms=([0-9.]+)",
|
||
"llm_total_ms": r"LLM_REQUEST_(?:DONE|FAILED)[^\n]*\btotal_ms=([0-9.]+)",
|
||
"start_run_total_seconds": r"\[start-run-timing\][^\n]*\btotal=([0-9.]+)s",
|
||
}
|
||
for name, pattern in numeric_patterns.items():
|
||
values = []
|
||
for raw in re.findall(pattern, text, flags=re.IGNORECASE):
|
||
with contextlib.suppress(ValueError):
|
||
values.append(float(raw))
|
||
if values:
|
||
timing_max[name] = max(timing_max.get(name, 0.0), max(values))
|
||
files.append(
|
||
{
|
||
"path": str(path),
|
||
"bytes_scanned": min(stat.st_size, tail_bytes),
|
||
"modified_at": dt.datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
|
||
"pattern_counts": counts,
|
||
}
|
||
)
|
||
return {"files": files, "pattern_counts": dict(totals), "timing_max": timing_max}
|
||
|
||
|
||
def collect_catalogue_scan(config_path: Path | None) -> dict[str, Any]:
|
||
if config_path is None:
|
||
return {"available": False, "reason": "config.yaml not found"}
|
||
runtime_home = next((path for path in _runtime_home_candidates(config_path) if path.exists()), config_path.parent / ".deer-flow")
|
||
skills_root = Path(os.environ.get("DEER_FLOW_SKILLS_PATH", str(config_path.parent / "skills")))
|
||
agents_root = Path(os.environ.get("DEER_FLOW_AGENTS_PATH", str(runtime_home / "agents")))
|
||
output: dict[str, Any] = {"available": True}
|
||
for label, root, names in (
|
||
("skills", skills_root, {"SKILL.md", "config.yaml"}),
|
||
("agents", agents_root, {"config.yaml", "SOUL.md"}),
|
||
):
|
||
started = time.perf_counter()
|
||
directories = 0
|
||
files = 0
|
||
bytes_total = 0
|
||
errors = 0
|
||
cap_reason: str | None = None
|
||
deadline = time.perf_counter() + 3.0
|
||
if root.is_dir():
|
||
try:
|
||
for current_root, dirnames, filenames in os.walk(root):
|
||
if time.perf_counter() >= deadline:
|
||
dirnames.clear()
|
||
cap_reason = "3s_deadline"
|
||
break
|
||
directories += len(dirnames)
|
||
if directories >= 20_000:
|
||
dirnames.clear()
|
||
cap_reason = "20000_directories"
|
||
break
|
||
for filename in filenames:
|
||
if filename not in names and not (label == "skills" and filename.endswith((".md", ".py"))):
|
||
continue
|
||
files += 1
|
||
with contextlib.suppress(OSError):
|
||
bytes_total += (Path(current_root) / filename).stat().st_size
|
||
if files >= 100_000:
|
||
cap_reason = "100000_files"
|
||
break
|
||
if files >= 100_000:
|
||
break
|
||
except OSError:
|
||
errors += 1
|
||
output[label] = {
|
||
"root": str(root),
|
||
"exists": root.is_dir(),
|
||
"directories": directories,
|
||
"matching_files": files,
|
||
"bytes": bytes_total,
|
||
"metadata_walk_ms": monotonic_ms(started),
|
||
"errors": errors,
|
||
"capped": cap_reason is not None,
|
||
"cap_reason": cap_reason,
|
||
}
|
||
return output
|
||
|
||
|
||
def _escape_mysql_option(value: str) -> str:
|
||
if "\n" in value or "\r" in value:
|
||
raise ValueError("MySQL option contains a newline")
|
||
return value.replace("\\", "\\\\").replace('"', '\\"')
|
||
|
||
|
||
def _mysql_url_parts(url: str) -> dict[str, Any]:
|
||
normalized = re.sub(r"^mysql(?:\+[A-Za-z0-9_]+)?://", "mysql://", url, count=1)
|
||
parsed = urllib.parse.urlsplit(normalized)
|
||
if parsed.scheme != "mysql" or not parsed.hostname:
|
||
raise ValueError("MySQL URL must use mysql:// or mysql+driver://")
|
||
if parsed.fragment:
|
||
raise ValueError("MySQL URL fragments are unsupported; use --mysql-defaults-file")
|
||
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True, strict_parsing=True)
|
||
normalized_query: dict[str, list[str]] = {}
|
||
for key, values in query.items():
|
||
normalized_key = key.lower().replace("_", "-")
|
||
if normalized_key in normalized_query:
|
||
raise ValueError("MySQL URL has duplicate normalized options; use --mysql-defaults-file")
|
||
normalized_query[normalized_key] = values
|
||
allowed = {"ssl-mode", "ssl-ca", "ssl-cert", "ssl-key"}
|
||
unknown = set(normalized_query) - allowed
|
||
if unknown or any(len(values) != 1 for values in normalized_query.values()):
|
||
raise ValueError("MySQL URL has unsupported or repeated options; use --mysql-defaults-file")
|
||
ssl_mode_values = normalized_query.get("ssl-mode") or []
|
||
ssl_mode = ssl_mode_values[0].upper() if ssl_mode_values else None
|
||
if ssl_mode not in {None, "REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"}:
|
||
raise ValueError("MySQL URL ssl-mode must be REQUIRED, VERIFY_CA, or VERIFY_IDENTITY")
|
||
output = {
|
||
"host": parsed.hostname,
|
||
"port": parsed.port or 3306,
|
||
"user": urllib.parse.unquote(parsed.username or ""),
|
||
"password": urllib.parse.unquote(parsed.password or ""),
|
||
"database": urllib.parse.unquote(parsed.path.lstrip("/")),
|
||
"ssl-mode": ssl_mode,
|
||
}
|
||
for name in ("ssl-ca", "ssl-cert", "ssl-key"):
|
||
values = normalized_query.get(name)
|
||
if values:
|
||
# parse_qs() already percent-decodes query values. Decoding a
|
||
# second time could change a literal "%2F" in a file name.
|
||
output[name] = values[0]
|
||
output["protocol"] = "TCP"
|
||
return output
|
||
|
||
|
||
MYSQL_SAFE_OPTIONS = {
|
||
"host",
|
||
"port",
|
||
"user",
|
||
"password",
|
||
"database",
|
||
"protocol",
|
||
"socket",
|
||
"ssl-mode",
|
||
"ssl-ca",
|
||
"ssl-capath",
|
||
"ssl-cert",
|
||
"ssl-key",
|
||
"ssl-cipher",
|
||
"tls-version",
|
||
"ssl-crl",
|
||
"ssl-crlpath",
|
||
}
|
||
|
||
|
||
def _decode_mysql_option_value(raw: str) -> str:
|
||
value = raw.strip()
|
||
if value[:1] in {'"', "'"}:
|
||
if len(value) < 2 or value[-1] != value[0]:
|
||
raise ValueError("unterminated quote in MySQL defaults file")
|
||
value = value[1:-1]
|
||
output: list[str] = []
|
||
index = 0
|
||
escapes = {"b": "\b", "t": "\t", "n": "\n", "r": "\r", "s": " ", "\\": "\\", '"': '"', "'": "'"}
|
||
while index < len(value):
|
||
if value[index] == "\\" and index + 1 < len(value):
|
||
index += 1
|
||
escaped = value[index]
|
||
output.append(escapes.get(escaped, f"\\{escaped}"))
|
||
else:
|
||
output.append(value[index])
|
||
index += 1
|
||
decoded = "".join(output)
|
||
if "\x00" in decoded or "\r" in decoded or "\n" in decoded:
|
||
raise ValueError("MySQL option contains a forbidden control character")
|
||
return decoded
|
||
|
||
|
||
def _read_safe_mysql_defaults(path: Path) -> dict[str, str]:
|
||
options: dict[str, str] = {}
|
||
section = ""
|
||
for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||
stripped = raw_line.strip()
|
||
if not stripped or stripped.startswith(("#", ";")):
|
||
continue
|
||
if stripped.lower().startswith("!include"):
|
||
raise ValueError("MySQL defaults file include directives are not allowed")
|
||
if stripped.startswith("[") and stripped.endswith("]"):
|
||
section = stripped[1:-1].strip().lower()
|
||
continue
|
||
if section != "client":
|
||
continue
|
||
key, separator, raw_value = stripped.partition("=")
|
||
if not separator:
|
||
raise ValueError(f"MySQL [client] option on line {line_number} must use key=value")
|
||
normalized_key = key.strip().lower().replace("_", "-")
|
||
if normalized_key not in MYSQL_SAFE_OPTIONS:
|
||
raise ValueError(f"unsafe or unsupported MySQL [client] option: {normalized_key}")
|
||
if normalized_key in options:
|
||
raise ValueError(f"duplicate MySQL [client] option: {normalized_key}")
|
||
options[normalized_key] = _decode_mysql_option_value(raw_value)
|
||
if not options:
|
||
raise ValueError("MySQL defaults file has no supported [client] options")
|
||
if options.get("port"):
|
||
try:
|
||
port = int(options["port"])
|
||
except ValueError as exc:
|
||
raise ValueError("MySQL port must be an integer") from exc
|
||
if not 1 <= port <= 65535:
|
||
raise ValueError("MySQL port must be between 1 and 65535")
|
||
options["port"] = str(port)
|
||
if options.get("protocol") and options["protocol"].upper() not in {"TCP", "SOCKET"}:
|
||
raise ValueError("MySQL protocol must be TCP or SOCKET")
|
||
if options.get("protocol"):
|
||
options["protocol"] = options["protocol"].upper()
|
||
if options.get("ssl-mode"):
|
||
ssl_mode = options["ssl-mode"].upper()
|
||
if ssl_mode not in {"REQUIRED", "VERIFY_CA", "VERIFY_IDENTITY"}:
|
||
raise ValueError("MySQL ssl-mode must be REQUIRED, VERIFY_CA, or VERIFY_IDENTITY")
|
||
options["ssl-mode"] = ssl_mode
|
||
return options
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _mysql_defaults_file(
|
||
*,
|
||
explicit_file: str | None,
|
||
mysql_url_env: str | None,
|
||
mysql_host: str | None,
|
||
mysql_port: int,
|
||
) -> Iterable[tuple[str | None, dict[str, Any] | None, str | None]]:
|
||
env_name = mysql_url_env or "MYSQL_DATABASE_URL"
|
||
connection: dict[str, Any]
|
||
options: dict[str, Any]
|
||
if explicit_file:
|
||
path = Path(explicit_file).expanduser()
|
||
if not path.is_file():
|
||
yield None, None, f"MySQL defaults file not found: {path}"
|
||
return
|
||
try:
|
||
options = _read_safe_mysql_defaults(path)
|
||
except (OSError, UnicodeError, ValueError) as exc:
|
||
yield None, None, f"could not safely parse MySQL defaults file: {exc}"
|
||
return
|
||
connection = {
|
||
"source": "sanitized_defaults_file",
|
||
"host": options.get("host") or mysql_host,
|
||
"port": int(options.get("port") or mysql_port),
|
||
"database": options.get("database"),
|
||
"ssl-mode": options.get("ssl-mode"),
|
||
"protocol": options.get("protocol"),
|
||
}
|
||
if os.name == "posix":
|
||
with contextlib.suppress(OSError):
|
||
mode = path.stat().st_mode & 0o777
|
||
if mode & 0o077:
|
||
connection["permissions_warning"] = f"source defaults file mode {mode:o} is group/world accessible"
|
||
protocol = str(options.get("protocol") or "").upper()
|
||
local_socket = bool(protocol == "SOCKET" or (os.name == "posix" and protocol != "TCP" and (options.get("socket") or str(options.get("host") or "").lower() in {"", "localhost"})))
|
||
connection["transport"] = "unix_socket" if local_socket else "tcp"
|
||
if not local_socket and not options.get("ssl-mode"):
|
||
yield (
|
||
None,
|
||
connection,
|
||
"MySQL TCP authentication requires ssl-mode=REQUIRED (or VERIFY_CA/VERIFY_IDENTITY) in the sanitized [client] file. Only TCP timing was collected.",
|
||
)
|
||
return
|
||
else:
|
||
raw_url = os.environ.get(env_name)
|
||
if not raw_url:
|
||
if mysql_host:
|
||
yield None, {"source": "host_only", "host": mysql_host, "port": mysql_port}, None
|
||
else:
|
||
yield None, None, f"{env_name} is not set and --mysql-defaults-file was not provided"
|
||
return
|
||
try:
|
||
options = _mysql_url_parts(raw_url)
|
||
except ValueError as exc:
|
||
yield None, None, str(exc)
|
||
return
|
||
connection = {key: value for key, value in options.items() if key in {"host", "port", "database", "ssl-mode"} and value not in (None, "")}
|
||
connection["source"] = f"env:{env_name}"
|
||
if not options.get("ssl-mode"):
|
||
yield (
|
||
None,
|
||
connection,
|
||
"MySQL URL authentication requires ?ssl-mode=REQUIRED (or VERIFY_CA/VERIFY_IDENTITY); otherwise use --mysql-defaults-file. Only TCP timing was collected.",
|
||
)
|
||
return
|
||
|
||
content = "[client]\n"
|
||
for key in sorted(MYSQL_SAFE_OPTIONS):
|
||
value = options.get(key)
|
||
if value in (None, ""):
|
||
continue
|
||
content += f'{key}="{_escape_mysql_option(str(value))}"\n'
|
||
file_path: str | None = None
|
||
try:
|
||
with tempfile.NamedTemporaryFile(
|
||
mode="w",
|
||
encoding="utf-8",
|
||
prefix="deerflow-mysql-",
|
||
suffix=".cnf",
|
||
delete=False,
|
||
) as handle:
|
||
file_path = handle.name
|
||
handle.write(content)
|
||
handle.flush()
|
||
os.fsync(handle.fileno())
|
||
os.chmod(file_path, 0o600)
|
||
if os.name == "posix" and (Path(file_path).stat().st_mode & 0o077):
|
||
yield None, None, "could not enforce 0600 permissions on temporary MySQL credentials"
|
||
return
|
||
yield file_path, connection, None
|
||
finally:
|
||
if file_path:
|
||
with contextlib.suppress(OSError):
|
||
Path(file_path).unlink()
|
||
|
||
|
||
def _clean_mysql_environment(secret_env_name: str | None) -> dict[str, str]:
|
||
clean_env = dict(os.environ)
|
||
for name in (
|
||
"MYSQL_PWD",
|
||
"MYSQL_HOST",
|
||
"MYSQL_TCP_PORT",
|
||
"MYSQL_UNIX_PORT",
|
||
"MYSQL_TEST_LOGIN_FILE",
|
||
"MYSQL_GROUP_SUFFIX",
|
||
secret_env_name,
|
||
):
|
||
if name:
|
||
clean_env.pop(name, None)
|
||
return clean_env
|
||
|
||
|
||
def _mysql_supports_no_login_paths(secret_env_name: str | None) -> bool:
|
||
result = _run_command(
|
||
["mysql", "--no-defaults", "--help"],
|
||
timeout=5.0,
|
||
env=_clean_mysql_environment(secret_env_name),
|
||
)
|
||
return b"--no-login-paths" in result.get("output", b"")
|
||
|
||
|
||
def _mysql_command(
|
||
defaults_file: str,
|
||
sql: str,
|
||
timeout: float = 8.0,
|
||
*,
|
||
secret_env_name: str | None = None,
|
||
no_login_paths_supported: bool,
|
||
) -> dict[str, Any]:
|
||
clean_env = _clean_mysql_environment(secret_env_name)
|
||
arguments = ["mysql"]
|
||
if no_login_paths_supported:
|
||
arguments.append("--no-login-paths")
|
||
else:
|
||
clean_env["MYSQL_TEST_LOGIN_FILE"] = f"{defaults_file}.disabled-login-path"
|
||
arguments.append(f"--defaults-file={defaults_file}")
|
||
arguments.extend(
|
||
[
|
||
"--batch",
|
||
"--raw",
|
||
"--skip-column-names",
|
||
"--connect-timeout=5",
|
||
"--execute",
|
||
sql,
|
||
]
|
||
)
|
||
return _run_command(
|
||
arguments,
|
||
timeout=timeout,
|
||
env=clean_env,
|
||
)
|
||
|
||
|
||
def _parse_key_value_tsv(raw: bytes) -> dict[str, Any]:
|
||
output: dict[str, Any] = {}
|
||
for line in raw.decode(errors="replace").splitlines():
|
||
key, separator, value = line.partition("\t")
|
||
if not separator:
|
||
continue
|
||
value = value.strip()
|
||
try:
|
||
parsed: Any = int(value)
|
||
except ValueError:
|
||
try:
|
||
parsed = float(value)
|
||
except ValueError:
|
||
parsed = value[:200]
|
||
output[key.strip()] = parsed
|
||
return output
|
||
|
||
|
||
def _tcp_connect_timing(host: str, port: int, timeout: float = 3.0) -> dict[str, Any]:
|
||
started = time.perf_counter()
|
||
try:
|
||
sock, details = _connect_socket(host, port, timeout=timeout, family="auto")
|
||
except ProbeFailure as exc:
|
||
return {"success": False, "error_phase": exc.phase, "error": str(exc), "total_ms": monotonic_ms(started)}
|
||
with contextlib.suppress(OSError):
|
||
sock.close()
|
||
return {
|
||
"success": True,
|
||
"dns_ms": details["dns_ms"],
|
||
"connect_ms": details["connect_ms"],
|
||
"peer_ip": details["peer_ip"],
|
||
"total_ms": monotonic_ms(started),
|
||
}
|
||
|
||
|
||
def collect_mysql_diagnostics(
|
||
*,
|
||
enabled: bool,
|
||
mysql_url_env: str | None,
|
||
defaults_file: str | None,
|
||
mysql_host: str | None,
|
||
mysql_port: int,
|
||
probes: int = 3,
|
||
) -> dict[str, Any]:
|
||
if not enabled:
|
||
return {"enabled": False}
|
||
with _mysql_defaults_file(
|
||
explicit_file=defaults_file,
|
||
mysql_url_env=mysql_url_env,
|
||
mysql_host=mysql_host,
|
||
mysql_port=mysql_port,
|
||
) as (resolved_file, connection, config_error):
|
||
output: dict[str, Any] = {"enabled": True, "connection": connection}
|
||
if connection and connection.get("host"):
|
||
output["tcp"] = _tcp_connect_timing(str(connection["host"]), int(connection.get("port") or mysql_port))
|
||
if config_error:
|
||
output["available"] = False
|
||
output["reason"] = config_error
|
||
return output
|
||
if shutil.which("mysql") is None:
|
||
output["available"] = False
|
||
output["reason"] = "mysql command-line client not found; TCP timing may still be present"
|
||
return output
|
||
if resolved_file is None:
|
||
output["available"] = False
|
||
output["reason"] = "credentials were not provided"
|
||
return output
|
||
|
||
no_login_paths_supported = _mysql_supports_no_login_paths(mysql_url_env)
|
||
output["client_option_isolation"] = {
|
||
"defaults_file_sanitized": True,
|
||
"login_paths_disabled": no_login_paths_supported,
|
||
"login_file_redirected_when_unsupported": not no_login_paths_supported,
|
||
"ambient_mysql_environment_removed": True,
|
||
}
|
||
|
||
def mysql_query(sql: str, timeout: float = 8.0) -> dict[str, Any]:
|
||
return _mysql_command(
|
||
resolved_file,
|
||
sql,
|
||
timeout=timeout,
|
||
secret_env_name=mysql_url_env,
|
||
no_login_paths_supported=no_login_paths_supported,
|
||
)
|
||
|
||
ping_ms: list[float] = []
|
||
ping_errors: list[str] = []
|
||
for _index in range(max(1, min(probes, 10))):
|
||
result = mysql_query("SELECT 1;", timeout=8)
|
||
if result.get("returncode") == 0:
|
||
ping_ms.append(float(result["elapsed_ms"]))
|
||
else:
|
||
ping_errors.append("timeout" if result.get("timed_out") else "query_failed")
|
||
output["select_1_ms"] = summarize_numbers(ping_ms)
|
||
output["select_1_errors"] = dict(Counter(ping_errors))
|
||
output["available"] = bool(ping_ms)
|
||
if not ping_ms:
|
||
output["reason"] = "all authenticated SELECT 1 probes failed; subsequent status queries were skipped"
|
||
return output
|
||
|
||
variables_sql = "SHOW GLOBAL VARIABLES WHERE Variable_name IN ('max_connections','wait_timeout','interactive_timeout','slow_query_log','long_query_time');"
|
||
status_sql = (
|
||
"SHOW GLOBAL STATUS WHERE Variable_name IN "
|
||
"('Threads_connected','Threads_running','Threads_created','Connections','Aborted_connects',"
|
||
"'Max_used_connections','Slow_queries','Questions','Uptime','Innodb_row_lock_current_waits',"
|
||
"'Innodb_row_lock_time','Innodb_row_lock_waits','Created_tmp_disk_tables','Created_tmp_tables',"
|
||
"'Innodb_buffer_pool_reads','Innodb_buffer_pool_read_requests');"
|
||
)
|
||
variables_result = mysql_query(variables_sql)
|
||
status_result = mysql_query(status_sql)
|
||
if variables_result.get("returncode") == 0:
|
||
output["variables"] = _parse_key_value_tsv(variables_result["output"])
|
||
else:
|
||
output["variables_error"] = "query_failed"
|
||
if status_result.get("returncode") == 0:
|
||
output["status"] = _parse_key_value_tsv(status_result["output"])
|
||
else:
|
||
output["status_error"] = "query_failed"
|
||
|
||
process_sql = (
|
||
"SELECT COUNT(*),"
|
||
"COALESCE(SUM(COMMAND='Sleep'),0),"
|
||
"COALESCE(SUM(COMMAND<>'Sleep'),0),"
|
||
"COALESCE(SUM(COMMAND<>'Sleep' AND TIME>=5),0),"
|
||
"COALESCE(SUM(COMMAND<>'Sleep' AND TIME>=30),0),"
|
||
"COALESCE(SUM(LOWER(COALESCE(STATE,'')) LIKE '%lock%'),0),"
|
||
"COALESCE(MAX(CASE WHEN COMMAND<>'Sleep' THEN TIME ELSE 0 END),0) "
|
||
"FROM information_schema.PROCESSLIST;"
|
||
)
|
||
process_result = mysql_query(process_sql)
|
||
if process_result.get("returncode") == 0:
|
||
fields = process_result["output"].decode(errors="replace").strip().split("\t")
|
||
if len(fields) == 7:
|
||
names = [
|
||
"total",
|
||
"sleeping",
|
||
"active",
|
||
"active_at_least_5s",
|
||
"active_at_least_30s",
|
||
"lock_state_count",
|
||
"max_active_seconds",
|
||
]
|
||
with contextlib.suppress(ValueError):
|
||
output["processlist"] = {name: int(value) for name, value in zip(names, fields)}
|
||
else:
|
||
output["processlist_error"] = "query_failed_or_permission_denied"
|
||
|
||
lock_queries = {
|
||
"data_lock_waits": "SELECT COUNT(*) FROM performance_schema.data_lock_waits;",
|
||
"pending_metadata_locks": ("SELECT COUNT(*) FROM performance_schema.metadata_locks WHERE LOCK_STATUS='PENDING';"),
|
||
}
|
||
locks: dict[str, Any] = {}
|
||
for name, sql in lock_queries.items():
|
||
result = mysql_query(sql)
|
||
if result.get("returncode") == 0:
|
||
raw = result["output"].decode(errors="replace").strip()
|
||
with contextlib.suppress(ValueError):
|
||
locks[name] = int(raw)
|
||
else:
|
||
locks[f"{name}_error"] = "unavailable_or_permission_denied"
|
||
output["locks"] = locks
|
||
|
||
digest_sql = "SELECT COUNT(*),COALESCE(ROUND(MAX(AVG_TIMER_WAIT)/1000000000,3),0),COALESCE(ROUND(MAX(MAX_TIMER_WAIT)/1000000000,3),0) FROM performance_schema.events_statements_summary_by_digest WHERE COUNT_STAR>0;"
|
||
digest_result = mysql_query(digest_sql)
|
||
if digest_result.get("returncode") == 0:
|
||
fields = digest_result["output"].decode(errors="replace").strip().split("\t")
|
||
if len(fields) == 3:
|
||
with contextlib.suppress(ValueError):
|
||
output["statement_digest_summary"] = {
|
||
"digests": int(fields[0]),
|
||
"max_average_ms": float(fields[1]),
|
||
"max_single_ms": float(fields[2]),
|
||
}
|
||
else:
|
||
output["statement_digest_error"] = "unavailable_or_permission_denied"
|
||
return output
|
||
|
||
|
||
def collect_sqlite_diagnostics(config_path: Path | None, config: Mapping[str, Any]) -> dict[str, Any]:
|
||
if str(config.get("database.backend") or "").lower() != "sqlite" or config_path is None:
|
||
return {"enabled": False}
|
||
import sqlite3
|
||
|
||
sqlite_dir = str(config.get("database.sqlite_dir") or ".deer-flow/data")
|
||
base = Path(sqlite_dir)
|
||
if not base.is_absolute():
|
||
base = config_path.parent / base
|
||
candidates = [base / "deerflow.db", base / "deerflow.sqlite", base / "deerflow.sqlite3"]
|
||
db_path = next((item for item in candidates if item.is_file()), None)
|
||
if db_path is None:
|
||
return {"enabled": True, "available": False, "reason": "SQLite database file not found"}
|
||
started = time.perf_counter()
|
||
try:
|
||
uri = f"file:{urllib.parse.quote(str(db_path))}?mode=ro"
|
||
connection = sqlite3.connect(uri, uri=True, timeout=0.5)
|
||
try:
|
||
connection.execute("SELECT 1").fetchone()
|
||
journal_mode = connection.execute("PRAGMA journal_mode").fetchone()[0]
|
||
page_count = int(connection.execute("PRAGMA page_count").fetchone()[0])
|
||
page_size = int(connection.execute("PRAGMA page_size").fetchone()[0])
|
||
finally:
|
||
connection.close()
|
||
except sqlite3.Error as exc:
|
||
return {
|
||
"enabled": True,
|
||
"available": False,
|
||
"path": str(db_path),
|
||
"error": _safe_error(exc),
|
||
"probe_ms": monotonic_ms(started),
|
||
}
|
||
return {
|
||
"enabled": True,
|
||
"available": True,
|
||
"path": str(db_path),
|
||
"probe_ms": monotonic_ms(started),
|
||
"journal_mode": journal_mode,
|
||
"page_count": page_count,
|
||
"page_size": page_size,
|
||
"database_bytes": db_path.stat().st_size,
|
||
"wal_bytes": db_path.with_name(db_path.name + "-wal").stat().st_size if db_path.with_name(db_path.name + "-wal").is_file() else 0,
|
||
"shm_bytes": db_path.with_name(db_path.name + "-shm").stat().st_size if db_path.with_name(db_path.name + "-shm").is_file() else 0,
|
||
}
|
||
|
||
|
||
def _json_request(
|
||
url: str,
|
||
*,
|
||
headers: Mapping[str, str],
|
||
timeout: float,
|
||
insecure: bool,
|
||
ca_file: str | None,
|
||
) -> tuple[int | None, float, Any | None, str | None]:
|
||
started = time.perf_counter()
|
||
try:
|
||
request = urllib.request.Request(url, headers=dict(headers), method="GET")
|
||
handlers: list[Any] = [urllib.request.ProxyHandler({}), _NoRedirectHandler()]
|
||
if urllib.parse.urlsplit(url).scheme == "https":
|
||
context = ssl._create_unverified_context() if insecure else ssl.create_default_context(cafile=ca_file) # noqa: SLF001
|
||
handlers.append(urllib.request.HTTPSHandler(context=context))
|
||
opener = urllib.request.build_opener(*handlers)
|
||
with opener.open(request, timeout=timeout) as response:
|
||
status = int(response.status)
|
||
body = response.read(4 * 1024 * 1024)
|
||
except urllib.error.HTTPError as exc:
|
||
return int(exc.code), monotonic_ms(started), None, f"HTTP {exc.code}"
|
||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
|
||
return None, monotonic_ms(started), None, _safe_error(exc)
|
||
try:
|
||
payload = json.loads(body)
|
||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||
return status, monotonic_ms(started), None, "response was not JSON"
|
||
return status, monotonic_ms(started), payload, None
|
||
|
||
|
||
def collect_admin_concurrency(
|
||
origin: str | None,
|
||
*,
|
||
token_env: str | None,
|
||
timeout: float,
|
||
insecure: bool,
|
||
ca_file: str | None,
|
||
) -> dict[str, Any]:
|
||
if not token_env:
|
||
return {"enabled": False, "reason": "--admin-token-env not supplied"}
|
||
token = os.environ.get(token_env)
|
||
if not token:
|
||
return {"enabled": True, "available": False, "reason": f"environment variable {token_env} is not set"}
|
||
if not origin:
|
||
return {"enabled": True, "available": False, "reason": "admin origin is unavailable"}
|
||
headers = {"Authorization": token if token.lower().startswith("bearer ") else f"Bearer {token}"}
|
||
paths = {
|
||
"active": "/api/admin/active-runs",
|
||
"concurrency": "/api/admin/active-runs/concurrency?granularity=minute&hours=1",
|
||
"token_speed": "/api/admin/active-runs/token-speed?granularity=minute&hours=1",
|
||
"limits": "/api/system-settings/model-concurrency",
|
||
}
|
||
output: dict[str, Any] = {
|
||
"enabled": True,
|
||
"available": False,
|
||
"origin": sanitize_origin(origin),
|
||
"proxy_mode": "direct",
|
||
}
|
||
successful_endpoints = 0
|
||
failed_endpoints: list[str] = []
|
||
required_keys = {
|
||
"active": {"count", "runs"},
|
||
"concurrency": {"enabled", "points"},
|
||
"token_speed": {"points"},
|
||
"limits": {"enabled", "default_limit", "user_default_limit", "model_limits", "user_model_limits"},
|
||
}
|
||
|
||
def safe_int(value: Any, default: int = 0) -> int:
|
||
try:
|
||
parsed = int(value)
|
||
except (TypeError, ValueError, OverflowError):
|
||
return default
|
||
return parsed
|
||
|
||
def safe_float(value: Any, default: float = 0.0) -> float:
|
||
try:
|
||
parsed = float(value)
|
||
except (TypeError, ValueError, OverflowError):
|
||
return default
|
||
return parsed if math.isfinite(parsed) else default
|
||
|
||
def payload_schema_ok(name: str, payload: Any) -> bool:
|
||
if not isinstance(payload, dict) or not required_keys[name].issubset(payload):
|
||
return False
|
||
if name == "active":
|
||
return isinstance(payload.get("count"), int) and isinstance(payload.get("runs"), list)
|
||
if name == "concurrency":
|
||
return isinstance(payload.get("enabled"), bool) and isinstance(payload.get("points"), list)
|
||
if name == "token_speed":
|
||
return isinstance(payload.get("points"), list)
|
||
return (
|
||
isinstance(payload.get("enabled"), bool)
|
||
and isinstance(payload.get("default_limit"), int)
|
||
and isinstance(payload.get("user_default_limit"), int)
|
||
and isinstance(payload.get("model_limits"), dict)
|
||
and isinstance(payload.get("user_model_limits"), dict)
|
||
)
|
||
|
||
for name, path in paths.items():
|
||
url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/"))
|
||
status, elapsed_ms, payload, error = _json_request(
|
||
url,
|
||
headers=headers,
|
||
timeout=timeout,
|
||
insecure=insecure,
|
||
ca_file=ca_file,
|
||
)
|
||
entry: dict[str, Any] = {"status_code": status, "elapsed_ms": elapsed_ms}
|
||
if error:
|
||
entry["error"] = error
|
||
endpoint_ok = status is not None and 200 <= status < 300 and payload_schema_ok(name, payload)
|
||
if endpoint_ok:
|
||
successful_endpoints += 1
|
||
else:
|
||
failed_endpoints.append(name)
|
||
if not error and status is not None and 200 <= status < 300:
|
||
entry["error"] = "unexpected JSON schema"
|
||
entry["available"] = endpoint_ok
|
||
if name == "active" and endpoint_ok:
|
||
runs = payload.get("runs") if isinstance(payload.get("runs"), list) else []
|
||
entry.update(
|
||
{
|
||
"count": safe_int(payload.get("count"), len(runs)),
|
||
"by_model": dict(Counter(str(run.get("model_name") or "unknown") for run in runs if isinstance(run, dict))),
|
||
"by_kind": dict(Counter(str(run.get("kind") or "unknown") for run in runs if isinstance(run, dict))),
|
||
"max_elapsed_seconds": max([safe_int(run.get("elapsed_seconds")) for run in runs if isinstance(run, dict)] or [0]),
|
||
}
|
||
)
|
||
elif name == "concurrency" and endpoint_ok:
|
||
points = payload.get("points") if isinstance(payload.get("points"), list) else []
|
||
entry.update(
|
||
{
|
||
"monitor_enabled": bool(payload.get("enabled")),
|
||
"points": len(points),
|
||
"peak": max([safe_int(point.get("peak")) for point in points if isinstance(point, dict)] or [0]),
|
||
"max_average": max([safe_float(point.get("avg")) for point in points if isinstance(point, dict)] or [0.0]),
|
||
}
|
||
)
|
||
elif name == "token_speed" and endpoint_ok:
|
||
points = payload.get("points") if isinstance(payload.get("points"), list) else []
|
||
speeds = [safe_float(point.get("avg_tokens_per_sec")) for point in points if isinstance(point, dict)]
|
||
entry.update(
|
||
{
|
||
"points": len(points),
|
||
"min_bucket_average_tokens_per_sec": min(speeds) if speeds else None,
|
||
"max_bucket_average_tokens_per_sec": max(speeds) if speeds else None,
|
||
}
|
||
)
|
||
elif name == "limits" and endpoint_ok:
|
||
safe_payload: dict[str, Any] = {}
|
||
for key in ("enabled", "default_limit", "user_default_limit", "model_limits", "user_model_limits"):
|
||
value = payload.get(key)
|
||
if key in {"model_limits", "user_model_limits"} and isinstance(value, dict):
|
||
safe_payload[key] = {str(model)[:120]: safe_int(limit) for model, limit in list(value.items())[:500] if safe_int(limit) > 0}
|
||
elif key == "enabled":
|
||
safe_payload[key] = bool(value)
|
||
elif key in {"default_limit", "user_default_limit"}:
|
||
parsed_limit = safe_int(value, -1)
|
||
safe_payload[key] = parsed_limit if parsed_limit >= 0 else None
|
||
entry["settings"] = safe_payload
|
||
output[name] = entry
|
||
output["available"] = successful_endpoints > 0
|
||
output["successful_endpoints"] = successful_endpoints
|
||
output["failed_endpoints"] = failed_endpoints
|
||
output["partial"] = 0 < successful_endpoints < len(paths)
|
||
if successful_endpoints == 0 and all(output[name].get("status_code") in {401, 403} for name in paths):
|
||
output["reason"] = "admin token was rejected"
|
||
elif successful_endpoints == 0:
|
||
output["reason"] = "no admin endpoint returned a 2xx JSON object"
|
||
return output
|
||
|
||
|
||
def run_deerflow_suite(origin: str, options: ProbeOptions, *, authenticated: bool) -> dict[str, Any]:
|
||
paths = [
|
||
("health", "/api/health", False),
|
||
("auth_options", "/api/v1/auth/options", False),
|
||
]
|
||
if authenticated:
|
||
paths.extend(
|
||
[
|
||
("auth_me", "/api/v1/auth/me", True),
|
||
("agents_selector", "/api/agents/selector", True),
|
||
(
|
||
"agents_mine",
|
||
"/api/agents?scope=mine&page=1&page_size=20&sort=time_desc",
|
||
True,
|
||
),
|
||
(
|
||
"agents_square",
|
||
"/api/agents?scope=square&page=1&page_size=20&sort=time_desc",
|
||
True,
|
||
),
|
||
("skills", "/api/skills?sort=time_desc", True),
|
||
]
|
||
)
|
||
output: dict[str, Any] = {}
|
||
public_headers: dict[str, str] = {}
|
||
for name, path, use_auth in paths:
|
||
suite_options = ProbeOptions(
|
||
method="GET",
|
||
headers=dict(options.headers if use_auth else public_headers),
|
||
connect_timeout=options.connect_timeout,
|
||
first_byte_timeout=options.first_byte_timeout,
|
||
read_timeout=options.read_timeout,
|
||
total_timeout=options.total_timeout,
|
||
max_response_bytes=options.max_response_bytes,
|
||
read_mode="full",
|
||
insecure=options.insecure,
|
||
ca_file=options.ca_file,
|
||
follow_redirects=False,
|
||
proxy=options.proxy,
|
||
use_env_proxy=options.use_env_proxy,
|
||
ip_family=options.ip_family,
|
||
)
|
||
url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/"))
|
||
result = probe_url(url, suite_options)
|
||
output[name] = {
|
||
"url": result["url"],
|
||
"success": result["success"],
|
||
"status_code": result["status_code"],
|
||
"error_phase": result["error_phase"],
|
||
"timings_ms": result["timings_ms"],
|
||
"body_bytes_read": result["hops"][-1].get("response", {}).get("body_bytes_read", 0),
|
||
}
|
||
return output
|
||
|
||
|
||
def run_health_baselines(origins: Sequence[str], options: ProbeOptions) -> dict[str, Any]:
|
||
paths = ("/health", "/api/health", "/api/langgraph/health")
|
||
output: dict[str, Any] = {}
|
||
public_headers: dict[str, str] = {}
|
||
health_options = ProbeOptions(
|
||
method="GET",
|
||
headers=public_headers,
|
||
connect_timeout=options.connect_timeout,
|
||
first_byte_timeout=min(options.first_byte_timeout, 10.0),
|
||
read_timeout=min(options.read_timeout, 3.0),
|
||
total_timeout=min(options.total_timeout, 15.0),
|
||
max_response_bytes=65536,
|
||
read_mode="full",
|
||
insecure=options.insecure,
|
||
ca_file=options.ca_file,
|
||
follow_redirects=False,
|
||
proxy=options.proxy,
|
||
use_env_proxy=options.use_env_proxy,
|
||
ip_family=options.ip_family,
|
||
)
|
||
for origin in dict.fromkeys(origins):
|
||
origin_output: dict[str, Any] = {}
|
||
for path in paths:
|
||
url = urllib.parse.urljoin(origin.rstrip("/") + "/", path.lstrip("/"))
|
||
result = probe_url(url, health_options)
|
||
origin_output[path] = {
|
||
"status_code": result["status_code"],
|
||
"success": result["success"],
|
||
"error_phase": result["error_phase"],
|
||
"timings_ms": result["timings_ms"],
|
||
}
|
||
output[sanitize_origin(origin)] = origin_output
|
||
return output
|
||
|
||
|
||
def run_interference_probe(
|
||
target_url: str,
|
||
sentinel_url: str,
|
||
target_options: ProbeOptions,
|
||
*,
|
||
interval: float,
|
||
max_sentinel_probes: int,
|
||
) -> dict[str, Any]:
|
||
"""Probe health repeatedly while one target request is in flight."""
|
||
stop = threading.Event()
|
||
sentinel_results: list[dict[str, Any]] = []
|
||
public_headers: dict[str, str] = {}
|
||
sentinel_options = ProbeOptions(
|
||
method="GET",
|
||
headers=public_headers,
|
||
connect_timeout=min(target_options.connect_timeout, 3.0),
|
||
first_byte_timeout=min(target_options.first_byte_timeout, 5.0),
|
||
read_timeout=min(target_options.read_timeout, 2.0),
|
||
total_timeout=min(target_options.total_timeout, 10.0),
|
||
max_response_bytes=65536,
|
||
read_mode="full",
|
||
insecure=target_options.insecure,
|
||
ca_file=target_options.ca_file,
|
||
proxy=target_options.proxy,
|
||
use_env_proxy=target_options.use_env_proxy,
|
||
ip_family=target_options.ip_family,
|
||
)
|
||
|
||
def sentinel_loop() -> None:
|
||
while not stop.is_set() and len(sentinel_results) < max_sentinel_probes:
|
||
sentinel_results.append(probe_url(sentinel_url, sentinel_options))
|
||
stop.wait(max(0.1, interval))
|
||
|
||
thread = threading.Thread(target=sentinel_loop, name="deerflow-health-sentinel", daemon=True)
|
||
started = time.perf_counter()
|
||
thread.start()
|
||
target_result = probe_url(target_url, target_options)
|
||
stop.set()
|
||
thread.join(timeout=max(1.0, target_options.read_timeout + 1.0))
|
||
return {
|
||
"target_url": sanitize_url(target_url),
|
||
"sentinel_url": sanitize_url(sentinel_url),
|
||
"wall_ms": monotonic_ms(started),
|
||
"target": target_result,
|
||
"sentinel_results": sentinel_results,
|
||
"sentinel_summary": summarize_probe_results(sentinel_results),
|
||
"caveat": "Multiple Gateway workers or load balancers can route sentinel requests around a blocked worker.",
|
||
}
|
||
|
||
|
||
def probe_nginx_stub_status(url: str | None, options: ProbeOptions) -> dict[str, Any]:
|
||
if not url:
|
||
return {"enabled": False}
|
||
request = urllib.request.Request(url, method="GET", headers={"User-Agent": USER_AGENT})
|
||
handlers: list[Any] = [urllib.request.ProxyHandler({}), _NoRedirectHandler()]
|
||
if urllib.parse.urlsplit(url).scheme == "https":
|
||
context = (
|
||
ssl._create_unverified_context() # noqa: SLF001
|
||
if options.insecure
|
||
else ssl.create_default_context(cafile=options.ca_file)
|
||
)
|
||
handlers.append(urllib.request.HTTPSHandler(context=context))
|
||
started = time.perf_counter()
|
||
try:
|
||
with urllib.request.build_opener(*handlers).open(request, timeout=options.first_byte_timeout) as response:
|
||
status = response.status
|
||
text = response.read(65536).decode(errors="replace")
|
||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
|
||
return {
|
||
"enabled": True,
|
||
"available": False,
|
||
"proxy_mode": "direct",
|
||
"url": sanitize_url(url),
|
||
"elapsed_ms": monotonic_ms(started),
|
||
"error": _safe_error(exc),
|
||
}
|
||
active_match = re.search(r"Active connections:\s*(\d+)", text)
|
||
state_match = re.search(r"Reading:\s*(\d+)\s+Writing:\s*(\d+)\s+Waiting:\s*(\d+)", text)
|
||
accepts_match = re.search(r"server accepts handled requests\s*\n\s*(\d+)\s+(\d+)\s+(\d+)", text)
|
||
return {
|
||
"enabled": True,
|
||
"available": bool(active_match and state_match),
|
||
"proxy_mode": "direct",
|
||
"url": sanitize_url(url),
|
||
"status_code": int(status),
|
||
"elapsed_ms": monotonic_ms(started),
|
||
"active_connections": int(active_match.group(1)) if active_match else None,
|
||
"reading": int(state_match.group(1)) if state_match else None,
|
||
"writing": int(state_match.group(2)) if state_match else None,
|
||
"waiting": int(state_match.group(3)) if state_match else None,
|
||
"accepted": int(accepts_match.group(1)) if accepts_match else None,
|
||
"handled": int(accepts_match.group(2)) if accepts_match else None,
|
||
"requests": int(accepts_match.group(3)) if accepts_match else None,
|
||
}
|
||
|
||
|
||
SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2, "ok": 3}
|
||
|
||
|
||
def _add_finding(
|
||
findings: list[dict[str, Any]],
|
||
severity: str,
|
||
area: str,
|
||
code: str,
|
||
title: str,
|
||
evidence: Sequence[str],
|
||
interpretation: str,
|
||
next_step: str,
|
||
) -> None:
|
||
findings.append(
|
||
{
|
||
"severity": severity,
|
||
"area": area,
|
||
"code": code,
|
||
"title": title,
|
||
"evidence": list(evidence),
|
||
"interpretation": interpretation,
|
||
"next_step": next_step,
|
||
}
|
||
)
|
||
|
||
|
||
def _summary_p50(summary: Mapping[str, Any], key: str) -> float | None:
|
||
value = summary.get("timings_ms", {}).get(key, {}).get("p50")
|
||
return float(value) if value is not None else None
|
||
|
||
|
||
def _target_response_signatures(target: Mapping[str, Any]) -> tuple[tuple[int, str], ...]:
|
||
signatures: set[tuple[int, str]] = set()
|
||
for sample in target.get("samples", []):
|
||
if not sample.get("success"):
|
||
continue
|
||
hops = sample.get("hops") or []
|
||
if not hops:
|
||
continue
|
||
hop = hops[-1]
|
||
status = hop.get("status_code")
|
||
if not isinstance(status, int):
|
||
continue
|
||
content_type = str(hop.get("response", {}).get("headers", {}).get("content-type") or "")
|
||
media_type = content_type.split(";", 1)[0].strip().lower()
|
||
signatures.add((status, media_type))
|
||
return tuple(sorted(signatures))
|
||
|
||
|
||
def _suite_total(suite: Mapping[str, Any], key: str) -> float | None:
|
||
value = suite.get(key, {}).get("timings_ms", {}).get("total")
|
||
return float(value) if value is not None else None
|
||
|
||
|
||
def _health_total_for_origin(health: Mapping[str, Any], origin: str, path: str = "/api/health") -> float | None:
|
||
value = health.get(sanitize_origin(origin), {}).get(path, {}).get("timings_ms", {}).get("total")
|
||
return float(value) if value is not None else None
|
||
|
||
|
||
def build_findings(report: Mapping[str, Any]) -> list[dict[str, Any]]:
|
||
findings: list[dict[str, Any]] = []
|
||
http_section = report.get("http", {})
|
||
targets = http_section.get("targets", [])
|
||
for target in targets:
|
||
label = str(target.get("label") or "target")
|
||
summary = target.get("summary", {})
|
||
errors = int(summary.get("transport_errors") or 0)
|
||
requests = int(summary.get("requests") or 0)
|
||
if errors:
|
||
_add_finding(
|
||
findings,
|
||
"critical" if errors == requests else "warning",
|
||
"network",
|
||
"transport_errors",
|
||
f"{label} 存在传输层失败",
|
||
[f"{errors}/{requests} 次失败", f"失败阶段={summary.get('error_phase_counts', {})}"],
|
||
"失败阶段可以先把问题定位到 DNS、TCP、TLS、首响应等待或正文读取；HTTP 4xx/5xx 不计为传输失败。",
|
||
"按报告中的 error_phase 处理；如果是 first_byte，继续对照直连 Gateway 与 /api/health。",
|
||
)
|
||
status_counts = summary.get("status_counts", {})
|
||
bad_gateway = sum(int(status_counts.get(str(code), 0)) for code in (502, 503, 504))
|
||
if bad_gateway:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"nginx",
|
||
"upstream_http_errors",
|
||
f"{label} 返回网关/上游错误",
|
||
[f"502/503/504 共 {bad_gateway} 次", f"状态分布={status_counts}"],
|
||
"代理成功接到请求，但上游连接、上游处理或代理超时失败。",
|
||
"对照 Nginx error log 的 upstream timed out/connect failed，并检查直连 Gateway 是否正常。",
|
||
)
|
||
if int(status_counts.get("429", 0)):
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"http_429",
|
||
f"{label} 触发限流或并发上限",
|
||
[f"429 次数={status_counts.get('429')}"],
|
||
"可能是 DeerFlow 模型并发限制、上游配额或其他限流；仅凭状态码不能区分。",
|
||
"查看管理员 active-runs/model-concurrency 数据和 Gateway 同时段日志。",
|
||
)
|
||
|
||
total = _summary_p50(summary, "total")
|
||
dns = _summary_p50(summary, "dns") or 0.0
|
||
tcp = _summary_p50(summary, "tcp_connect") or 0.0
|
||
tls = _summary_p50(summary, "tls_handshake") or 0.0
|
||
response_wait = _summary_p50(summary, "response_wait") or 0.0
|
||
body_read = _summary_p50(summary, "body_read") or 0.0
|
||
if dns >= 200:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"network",
|
||
"slow_dns",
|
||
f"{label} DNS 解析慢",
|
||
[f"DNS p50={dns:.1f}ms"],
|
||
"连接尚未到达 Nginx 或 Gateway，先查 DNS/解析链路。",
|
||
"在客户端和服务器分别检查 resolv.conf、DNS 服务器延迟与缓存。",
|
||
)
|
||
if tcp >= 300:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"network",
|
||
"slow_tcp_connect",
|
||
f"{label} TCP 建连慢",
|
||
[f"TCP p50={tcp:.1f}ms"],
|
||
"延迟发生在应用收到 HTTP 请求前，可能涉及网络、SYN 队列、防火墙或连接耗尽。",
|
||
"查看 ss 监听队列/SYN-RECV，并从同网段及服务器本机分别复测。",
|
||
)
|
||
if tls >= 500:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"network",
|
||
"slow_tls",
|
||
f"{label} TLS 握手慢",
|
||
[f"TLS p50={tls:.1f}ms"],
|
||
"延迟主要在证书/加密握手或中间代理，尚未进入业务处理。",
|
||
"检查 TLS 终止层 CPU、证书链、OCSP/中间安全设备，并与 HTTP 直连作对照。",
|
||
)
|
||
if total and response_wait >= 800 and response_wait / max(total, 1.0) >= 0.55:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"application",
|
||
"response_wait_dominates",
|
||
f"{label} 主要卡在请求发完到首响应之间",
|
||
[f"总耗时 p50={total:.1f}ms", f"response_wait p50={response_wait:.1f}ms"],
|
||
"请求已建连并上传，等待可能来自 Nginx 上游排队、Gateway 事件循环、数据库、文件扫描或下游服务；这不是单独证明 MySQL 慢。",
|
||
"结合直连、并行 health、MySQL processlist/locks 和日志中的分阶段计时继续定位。",
|
||
)
|
||
if body_read >= 1000 and response_wait < body_read:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"network",
|
||
"slow_body_transfer",
|
||
f"{label} 首响应后正文读取更慢",
|
||
[f"body_read p50={body_read:.1f}ms", f"response_wait p50={response_wait:.1f}ms"],
|
||
"更像大响应、慢流式输出、网络吞吐或代理缓冲，而不是请求开始前排队。",
|
||
"检查响应字节数、Content-Type、Nginx buffering 与客户端渲染；SSE 用 --read-mode first-byte。",
|
||
)
|
||
|
||
if len(targets) >= 2:
|
||
public = targets[0]
|
||
public_p50 = _summary_p50(public.get("summary", {}), "total")
|
||
public_signature = _target_response_signatures(public)
|
||
for compare in targets[1:]:
|
||
direct_p50 = _summary_p50(compare.get("summary", {}), "total")
|
||
compare_signature = _target_response_signatures(compare)
|
||
gap_is_large = bool(public_p50 and direct_p50 and public_p50 >= direct_p50 * 1.5 and public_p50 - direct_p50 >= 200)
|
||
responses_are_comparable = bool(
|
||
len(public_signature) == 1
|
||
and public_signature == compare_signature
|
||
and 200 <= public_signature[0][0] < 300
|
||
)
|
||
if gap_is_large and responses_are_comparable:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"nginx",
|
||
"public_direct_gap",
|
||
"公网/代理路径明显慢于对照地址",
|
||
[
|
||
f"{public.get('label')} p50={public_p50:.1f}ms",
|
||
f"{compare.get('label')} p50={direct_p50:.1f}ms",
|
||
],
|
||
"差值位于两条路径不同的网络、TLS、负载均衡或 Nginx 层；前提是两个 URL 实际执行同一业务。",
|
||
"确认 Host/路径/请求体一致，再检查 Nginx 连接、错误日志和上游 keepalive。",
|
||
)
|
||
elif gap_is_large and not responses_are_comparable:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"nginx",
|
||
"public_direct_not_comparable",
|
||
"公网与对照地址响应不一致，不能据此归因 Nginx",
|
||
[f"公网响应签名={public_signature}", f"对照响应签名={compare_signature}"],
|
||
"两条路径的状态码或 Content-Type 不同，快速的 401/404/HTML 页面不能作为同一业务的直连基线。",
|
||
"先确认 Host、路由、鉴权和路径一致，再进行耗时对比。",
|
||
)
|
||
|
||
local_gateway = http_section.get("local_gateway_health", {})
|
||
target_host_confirmed = bool(report.get("server", {}).get("scope", {}).get("target_host_confirmed"))
|
||
if target_host_confirmed and targets and local_gateway.get("success") and local_gateway.get("status_code") == 200:
|
||
primary_origin = _origin_for_request(str(targets[0].get("url") or ""))
|
||
public_health = http_section.get("health_baselines", {}).get(sanitize_origin(primary_origin), {}).get("/api/health", {})
|
||
public_health_ms = public_health.get("timings_ms", {}).get("total")
|
||
local_health_ms = local_gateway.get("timings_ms", {}).get("total")
|
||
if (
|
||
public_health.get("success")
|
||
and public_health.get("status_code") == 200
|
||
and public_health_ms is not None
|
||
and local_health_ms is not None
|
||
and float(public_health_ms) >= float(local_health_ms) * 1.5
|
||
and float(public_health_ms) - float(local_health_ms) >= 200
|
||
):
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"nginx",
|
||
"public_local_health_gap",
|
||
"公网 health 明显慢于本机 DeerFlow Gateway health",
|
||
[f"公网 /api/health={float(public_health_ms):.1f}ms", f"本机 /api/health={float(local_health_ms):.1f}ms"],
|
||
"两个 200 health 响应执行同一轻量路径，差值更偏向入口网络、TLS、负载均衡或 Nginx。",
|
||
"结合 Nginx 活跃连接、监听队列、FD 和错误日志判断是否连接容量或代理层问题。",
|
||
)
|
||
|
||
suite = http_section.get("deerflow_suite", {})
|
||
if suite:
|
||
health_ms = _suite_total(suite, "health")
|
||
options_ms = _suite_total(suite, "auth_options")
|
||
me_ms = _suite_total(suite, "auth_me")
|
||
agents_ms = max([_suite_total(suite, name) or 0.0 for name in ("agents_selector", "agents_mine", "agents_square")] or [0.0])
|
||
skills_ms = _suite_total(suite, "skills")
|
||
db_paths = [value for value in (options_ms, me_ms, agents_ms) if value is not None and value > 0]
|
||
if health_ms is not None and health_ms < 500 and db_paths and sum(value >= 1500 for value in db_paths) >= 2:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"mysql",
|
||
"health_fast_db_paths_slow",
|
||
"静态 health 快，但多条数据库相关接口慢",
|
||
[
|
||
f"health={health_ms:.1f}ms",
|
||
f"auth/options={options_ms}ms",
|
||
f"auth/me={me_ms}ms",
|
||
f"agents最大={agents_ms:.1f}ms",
|
||
],
|
||
"这是业务数据库、连接池、锁或数据库磁盘的强关联证据；auth/options、auth/me、agents 都会查业务库。",
|
||
"立即在卡顿时抓 MySQL Threads/PROCESSLIST/data_lock_waits/metadata_locks，并核对池上限。",
|
||
)
|
||
if health_ms is not None and health_ms < 500 and skills_ms and skills_ms >= 1500:
|
||
others = [value for value in (options_ms, me_ms, agents_ms) if value is not None and value > 0]
|
||
if not others or max(others) < skills_ms * 0.6:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"skills_path_specific",
|
||
"技能列表明显慢于健康和其他业务接口",
|
||
[f"health={health_ms:.1f}ms", f"skills={skills_ms:.1f}ms"],
|
||
"当前实现每次 /api/skills 都同步扫描并解析技能目录，再查所有权/收藏/标签；慢磁盘或技能很多会放大。",
|
||
"运行 interference 探针；若 health 同时尖峰，优先查同步文件扫描阻塞单 worker 事件循环。",
|
||
)
|
||
if health_ms is not None and health_ms < 500 and agents_ms >= 1500:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"deerflow",
|
||
"agents_path_slow",
|
||
"Agents 接口慢但静态 health 正常",
|
||
[f"health={health_ms:.1f}ms", f"agents最大={agents_ms:.1f}ms"],
|
||
"可能是首次/每小时 legacy agent 目录同步或 agents 多次串行 DB 查询；需看同一时段日志。",
|
||
"grep [start-run-timing] 和 agents 同步日志，并在第二次立即复测判断是否只有冷启动/TTL 首次慢。",
|
||
)
|
||
|
||
interference = http_section.get("interference")
|
||
if isinstance(interference, dict):
|
||
target_total = interference.get("target", {}).get("timings_ms", {}).get("total")
|
||
sentinel_p95 = interference.get("sentinel_summary", {}).get("timings_ms", {}).get("total", {}).get("p95")
|
||
sentinel_count = interference.get("sentinel_summary", {}).get("successful_transport", 0)
|
||
if target_total and float(target_total) >= 1000 and sentinel_count and sentinel_p95 is not None:
|
||
if float(sentinel_p95) >= 500:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"health_stalls_with_target",
|
||
"慢接口执行期间 health 也出现明显延迟",
|
||
[f"目标总耗时={float(target_total):.1f}ms", f"并行 health p95={float(sentinel_p95):.1f}ms"],
|
||
"强烈指向共享 Gateway worker/事件循环、CPU 或宿主 IO 阻塞；单 worker 下同步技能目录扫描尤其可疑。多 worker/负载均衡可能掩盖该现象。",
|
||
"对照进程 CPU/IO PSI、技能目录遍历时间和 Gateway 日志；不要先盲目扩大 MySQL 连接。",
|
||
)
|
||
elif float(sentinel_p95) < 250:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"application",
|
||
"health_fast_during_target",
|
||
"慢接口执行期间 health 保持快速",
|
||
[f"目标总耗时={float(target_total):.1f}ms", f"并行 health p95={float(sentinel_p95):.1f}ms"],
|
||
"更偏向该请求自己的数据库/下游服务/模型等待，而不是整个 Gateway 事件循环被卡死。",
|
||
"继续对照 auth/me、MySQL 和该接口专属下游；多 worker 环境下此结论只能算辅助证据。",
|
||
)
|
||
|
||
server = report.get("server", {})
|
||
easy_mode = bool(report.get("request", {}).get("easy_mode"))
|
||
target_host_confirmed = bool(server.get("scope", {}).get("target_host_confirmed"))
|
||
if easy_mode and not target_host_confirmed:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"resources",
|
||
"execution_host_not_confirmed",
|
||
"未确认脚本执行机就是目标 DeerFlow 服务器",
|
||
["本机未发现经过身份校验的 loopback DeerFlow Gateway"],
|
||
"已采集的 CPU、内存、Docker、Nginx 等数据属于脚本执行机，不能直接用于判断目标服务器配置不足。",
|
||
"把脚本复制到实际部署服务器运行；网络接口计时结果仍然有效。",
|
||
)
|
||
server = {}
|
||
socket_info = server.get("sockets", {})
|
||
for listener in socket_info.get("listeners", []):
|
||
usage = listener.get("backlog_usage_pct")
|
||
if usage is not None and float(usage) >= 70:
|
||
_add_finding(
|
||
findings,
|
||
"critical" if float(usage) >= 95 else "warning",
|
||
"nginx",
|
||
"listen_backlog_pressure",
|
||
f"端口 {listener.get('port')} 监听队列接近上限",
|
||
[
|
||
f"Recv-Q={listener.get('receive_queue')}",
|
||
f"backlog={listener.get('backlog')}",
|
||
f"占用={float(usage):.1f}%",
|
||
],
|
||
"新 TCP 连接可能在内核/Nginx/Gateway 接收前排队或丢弃。",
|
||
"核对 worker 是否处理不过来、somaxconn/tcp_max_syn_backlog、CPU 和文件句柄；先解决处理能力而非只放大 backlog。",
|
||
)
|
||
global_states = socket_info.get("global_states", {})
|
||
if int(global_states.get("SYN-RECV", 0)) >= 100:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"network",
|
||
"many_syn_recv",
|
||
"存在大量半连接",
|
||
[f"SYN-RECV={global_states.get('SYN-RECV')}"],
|
||
"可能是连接洪峰、backlog 压力或异常客户端。",
|
||
"按监听端口查看 ss -ant state syn-recv，并检查负载均衡、SYN cookies 与网络来源。",
|
||
)
|
||
|
||
nginx_status = server.get("nginx_status", {})
|
||
nginx_config = server.get("nginx", {})
|
||
if nginx_status.get("available"):
|
||
capacities = [int(item.get("theoretical_connection_slots")) for item in nginx_config.get("instances", []) if item.get("theoretical_connection_slots")]
|
||
capacity = min(capacities) if capacities else None
|
||
active = nginx_status.get("active_connections")
|
||
if capacity and active is not None and int(active) >= capacity * 0.7:
|
||
_add_finding(
|
||
findings,
|
||
"critical" if int(active) >= capacity * 0.9 else "warning",
|
||
"nginx",
|
||
"nginx_connection_slots",
|
||
"Nginx 活跃连接接近理论 worker_connections 上限",
|
||
[f"active={active}", f"理论槽位={capacity}"],
|
||
"反向代理通常每个客户端请求还占一个上游连接，实际可承载客户端数低于理论槽位。",
|
||
"先确认连接是否被长 SSE/keepalive 占用，再评估 worker_connections、worker_rlimit_nofile 和 worker 数。",
|
||
)
|
||
|
||
logs = server.get("logs", {})
|
||
pattern_counts = logs.get("pattern_counts", {})
|
||
log_finding_map = {
|
||
"nginx_connection_capacity": ("critical", "nginx", "Nginx 日志明确出现 worker_connections 不足"),
|
||
"too_many_open_files": ("critical", "resources", "日志出现文件句柄耗尽"),
|
||
"nginx_upstream_timeout": ("warning", "nginx", "Nginx 记录上游超时"),
|
||
"db_pool_timeout": ("critical", "mysql", "日志记录数据库连接池等待超时"),
|
||
"db_lock_or_deadlock": ("warning", "mysql", "日志记录数据库锁等待或死锁"),
|
||
"deerflow_concurrency_reject": ("warning", "deerflow", "日志记录 DeerFlow 并发拒绝"),
|
||
"resource_exhaustion": ("critical", "resources", "日志记录系统资源耗尽"),
|
||
}
|
||
for key, (severity, area, title) in log_finding_map.items():
|
||
count = int(pattern_counts.get(key, 0))
|
||
if count:
|
||
_add_finding(
|
||
findings,
|
||
severity,
|
||
area,
|
||
f"log_{key}",
|
||
title,
|
||
[f"最近日志尾部命中 {count} 次"],
|
||
"这是实际日志证据，但需按日志时间确认是否与本次卡顿同一时段。",
|
||
"打开原日志核对时间、请求和完整堆栈；本报告为防泄密只保存模式计数，不保存原行。",
|
||
)
|
||
|
||
mysql = server.get("mysql", {})
|
||
if mysql.get("available"):
|
||
ping_p50 = mysql.get("select_1_ms", {}).get("p50")
|
||
if ping_p50 is not None and float(ping_p50) >= 100:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"mysql",
|
||
"mysql_ping_slow",
|
||
"MySQL 新连接 + SELECT 1 延迟偏高",
|
||
[f"p50={float(ping_p50):.1f}ms"],
|
||
"包含 mysql 客户端启动、DNS、TCP、认证和查询，说明 DB 路径值得优先调查，但不等同于业务 SQL 本身慢。",
|
||
"在应用连接池内复测并抓 PROCESSLIST/慢查询；同时看 TCP timing 区分网络。",
|
||
)
|
||
mysql_status = mysql.get("status", {})
|
||
variables = mysql.get("variables", {})
|
||
connected = int(mysql_status.get("Threads_connected") or 0)
|
||
max_connections = int(variables.get("max_connections") or 0)
|
||
if max_connections and connected >= max_connections * 0.8:
|
||
_add_finding(
|
||
findings,
|
||
"critical" if connected >= max_connections * 0.95 else "warning",
|
||
"mysql",
|
||
"mysql_connections_high",
|
||
"MySQL 当前连接数接近 max_connections",
|
||
[f"Threads_connected={connected}", f"max_connections={max_connections}"],
|
||
"新请求可能在应用池或 MySQL 握手阶段等待/失败。",
|
||
"检查各 Gateway worker 的池总上限、Sleep 连接、泄漏和其他应用连接；不要只盲目调大 max_connections。",
|
||
)
|
||
processlist = mysql.get("processlist", {})
|
||
if int(processlist.get("lock_state_count") or 0) or int(processlist.get("active_at_least_5s") or 0):
|
||
_add_finding(
|
||
findings,
|
||
"critical" if int(processlist.get("active_at_least_30s") or 0) else "warning",
|
||
"mysql",
|
||
"mysql_active_waits",
|
||
"MySQL 当前存在长执行或锁状态会话",
|
||
[
|
||
f">=5s active={processlist.get('active_at_least_5s', 0)}",
|
||
f">=30s active={processlist.get('active_at_least_30s', 0)}",
|
||
f"lock state={processlist.get('lock_state_count', 0)}",
|
||
f"max active={processlist.get('max_active_seconds', 0)}s",
|
||
],
|
||
"这是卡顿时的实时数据库证据，可能直接拖慢 login/agents/skills 的串行查询。",
|
||
"由 DBA 查看对应 PROCESSLIST 详情、执行计划和事务持锁者；本脚本不保存 SQL 正文。",
|
||
)
|
||
locks = mysql.get("locks", {})
|
||
if int(locks.get("data_lock_waits") or 0) or int(locks.get("pending_metadata_locks") or 0):
|
||
_add_finding(
|
||
findings,
|
||
"critical",
|
||
"mysql",
|
||
"mysql_lock_waits",
|
||
"MySQL performance_schema 显示锁等待",
|
||
[
|
||
f"data_lock_waits={locks.get('data_lock_waits', 0)}",
|
||
f"pending_metadata_locks={locks.get('pending_metadata_locks', 0)}",
|
||
],
|
||
"数据锁或 metadata lock 可让相关查询成批等待；skills 路径的建表 checkfirst 尤其应对照 metadata lock。",
|
||
"立即保留锁等待链、持锁事务和表名证据，再由 DBA 处理；避免无证据重启掩盖现场。",
|
||
)
|
||
|
||
config = report.get("deerflow", {}).get("config", {})
|
||
backend = str(config.get("database.backend") or "").lower()
|
||
if backend == "mysql" and mysql.get("available"):
|
||
pool_size = int(config.get("database.pool_size") or 0)
|
||
overflow = int(config.get("database.max_overflow") or 0)
|
||
gateway_workers = int(server.get("host", {}).get("processes", {}).get("deerflow_gateway", {}).get("processes") or 1)
|
||
max_connections = int(mysql.get("variables", {}).get("max_connections") or 0)
|
||
theoretical_pool = (pool_size + overflow) * max(1, gateway_workers)
|
||
if max_connections and theoretical_pool >= max_connections * 0.8:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"mysql",
|
||
"pool_budget_risk",
|
||
"Gateway 理论连接池总量接近 MySQL 上限",
|
||
[
|
||
f"(pool_size {pool_size} + overflow {overflow}) × worker {gateway_workers} = {theoretical_pool}",
|
||
f"max_connections={max_connections}",
|
||
],
|
||
"还未计入其他应用、运维和 MySQL 保留连接，高峰时可能先在池或数据库侧排队。",
|
||
"为所有客户端预留余量，结合真实 Max_used_connections/Threads_connected 调整；先排查泄漏。",
|
||
)
|
||
|
||
host = server.get("host", {})
|
||
if host.get("supported"):
|
||
cpu_count = int(host.get("cpu_count") or 1)
|
||
cpu = host.get("cpu", {})
|
||
busy = cpu.get("busy_pct")
|
||
iowait = cpu.get("iowait_pct")
|
||
load = host.get("load_average") or []
|
||
if busy is not None and float(busy) >= 90:
|
||
_add_finding(
|
||
findings,
|
||
"critical",
|
||
"resources",
|
||
"cpu_saturated",
|
||
"服务器 CPU 已接近饱和",
|
||
[f"CPU busy={float(busy):.1f}%", f"CPU cores={cpu_count}", f"load={load}"],
|
||
"Gateway、TLS、Python 同步扫描和数据库客户端都可能排队。",
|
||
"用 pidstat/top 定位进程和线程；若是技能扫描先修热点，不要只扩机器。",
|
||
)
|
||
elif load and float(load[0]) >= cpu_count * 1.2:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"resources",
|
||
"load_high",
|
||
"系统 1 分钟负载高于 CPU 容量",
|
||
[f"load1={float(load[0]):.2f}", f"CPU cores={cpu_count}"],
|
||
"可运行任务或不可中断 IO 等待过多，接口可能在调度层排队。",
|
||
"结合 CPU busy、iowait、PSI 和进程 CPU 判断是算力还是 IO。",
|
||
)
|
||
if iowait is not None and float(iowait) >= 15:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"resources",
|
||
"cpu_iowait_high",
|
||
"CPU iowait 偏高",
|
||
[f"iowait={float(iowait):.1f}%"],
|
||
"磁盘或网络块设备等待可能拖慢 MySQL、SQLite、技能目录扫描与日志。",
|
||
"用 iostat -xz 1 10 对准 MySQL/技能所在盘确认 await 和 util。",
|
||
)
|
||
memory = host.get("memory", {})
|
||
available_pct = memory.get("available_pct")
|
||
if available_pct is not None and float(available_pct) < 10:
|
||
_add_finding(
|
||
findings,
|
||
"critical",
|
||
"resources",
|
||
"memory_low",
|
||
"可用内存不足",
|
||
[f"MemAvailable={float(available_pct):.1f}%", f"Swap used={memory.get('swap_used_pct')}%"],
|
||
"回收、换页或 OOM 会造成所有接口抖动，重启后短暂变快也与此相符。",
|
||
"定位 RSS 最大进程、容器内存限制和 OOM 记录；为模型/数据库/文件缓存保留余量。",
|
||
)
|
||
disk_util = host.get("disk_io", {}).get("max_util_pct")
|
||
if disk_util is not None and float(disk_util) >= 85:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"resources",
|
||
"disk_busy",
|
||
"块设备采样期间接近满负载",
|
||
[
|
||
f"device={host.get('disk_io', {}).get('busiest_device')}",
|
||
f"util={float(disk_util):.1f}%",
|
||
],
|
||
"可能拖慢数据库和技能/Agent 文件扫描；一次 1 秒采样只代表当前窗口。",
|
||
"卡顿持续期间运行 iostat -xz 1 10，确认 r_await/w_await 与具体挂载点。",
|
||
)
|
||
for path, usage in host.get("disk_space", {}).items():
|
||
free_pct = usage.get("free_pct")
|
||
if free_pct is not None and float(free_pct) < 10:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"resources",
|
||
"disk_space_low",
|
||
f"文件系统 {path} 剩余空间不足",
|
||
[f"free={float(free_pct):.1f}%"],
|
||
"数据库临时表、WAL、日志和容器写层可能受影响。",
|
||
"清理前先确认数据归属和备份；不要直接删除数据库/卷。",
|
||
)
|
||
for role, process in host.get("processes", {}).items():
|
||
fd_usage = process.get("max_fd_usage_pct")
|
||
if fd_usage is not None and float(fd_usage) >= 80:
|
||
_add_finding(
|
||
findings,
|
||
"critical" if float(fd_usage) >= 95 else "warning",
|
||
"resources",
|
||
"process_fd_high",
|
||
f"{role} 文件句柄接近进程上限",
|
||
[f"最大 FD 占用={float(fd_usage):.1f}%", f"进程数={process.get('processes')}"],
|
||
"新 socket、日志或文件打开会失败，表现为连接卡顿/502。",
|
||
"查 /proc/<pid>/fd 类型与泄漏，结合 limits/worker_connections 调整。",
|
||
)
|
||
if cpu_count <= 2:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"resources",
|
||
"small_cpu_capacity",
|
||
"服务器 CPU 核数较少",
|
||
[f"CPU cores={cpu_count}"],
|
||
"这不是当前卡顿的直接证据，但单 worker Gateway、Nginx、数据库或容器共享时余量有限。",
|
||
"先用实测 CPU/PSI/并发退化证明容量瓶颈，再决定扩容。",
|
||
)
|
||
|
||
docker = server.get("docker", {})
|
||
for container in docker.get("containers", []):
|
||
limits = container.get("limits", {})
|
||
stats = container.get("stats", {})
|
||
if limits.get("oom_killed"):
|
||
_add_finding(
|
||
findings,
|
||
"critical",
|
||
"resources",
|
||
"container_oom",
|
||
f"容器 {container.get('name')} 曾被 OOM kill",
|
||
[f"restart_count={limits.get('restart_count')}"],
|
||
"内存限制或宿主内存不足已经造成服务中断。",
|
||
"查 docker events/kernel OOM 时间和容器 RSS，调整限制或修复内存增长。",
|
||
)
|
||
if stats.get("memory_pct") is not None and float(stats["memory_pct"]) >= 90:
|
||
_add_finding(
|
||
findings,
|
||
"critical",
|
||
"resources",
|
||
"container_memory_high",
|
||
f"容器 {container.get('name')} 内存接近限制",
|
||
[f"memory={stats.get('memory_usage')}", f"memory_pct={stats.get('memory_pct')}%"],
|
||
"容器回收/OOM 风险可拖慢或重启服务。",
|
||
"结合宿主内存和容器内进程 RSS判断是否扩限或修复增长。",
|
||
)
|
||
|
||
catalogue = server.get("catalogue_scan", {})
|
||
skills = catalogue.get("skills", {})
|
||
if skills.get("exists") and float(skills.get("metadata_walk_ms") or 0) >= 500:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"skill_directory_walk_slow",
|
||
"技能目录仅元数据遍历就很慢",
|
||
[
|
||
f"目录={skills.get('directories')}",
|
||
f"文件={skills.get('matching_files')}",
|
||
f"walk={float(skills.get('metadata_walk_ms')):.1f}ms",
|
||
],
|
||
"/api/skills 会进一步读取和 YAML 解析每个技能文件，实际阻塞可能更长。",
|
||
"检查技能目录是否位于慢盘/NFS、是否残留过多文件，并用 interference 探针确认事件循环影响。",
|
||
)
|
||
elif int(skills.get("matching_files") or 0) >= 500:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"deerflow",
|
||
"large_skill_catalogue",
|
||
"技能目录文件量较大",
|
||
[f"匹配文件={skills.get('matching_files')}", f"metadata walk={skills.get('metadata_walk_ms')}ms"],
|
||
"当前 /api/skills 每请求全量扫描，前端重复刷新会放大成本；文件量本身不是卡顿证明。",
|
||
"对照访问日志中的 /api/skills 请求簇与 SKILL_VIEW_CACHE/health 干扰数据。",
|
||
)
|
||
|
||
admin = report.get("deerflow", {}).get("admin_concurrency", {})
|
||
if admin.get("available"):
|
||
active = admin.get("active", {})
|
||
count = int(active.get("count") or 0)
|
||
max_elapsed = int(active.get("max_elapsed_seconds") or 0)
|
||
limits = admin.get("limits", {}).get("settings", {})
|
||
default_limit = int(limits.get("default_limit") or 0)
|
||
if count and default_limit and count >= default_limit:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"active_runs_at_limit",
|
||
"活动运行数已达到默认模型并发限制",
|
||
[f"active={count}", f"default_limit={default_limit}", f"max_elapsed={max_elapsed}s"],
|
||
"新 Agent 请求可能收到 429；这不是 Nginx 排队，而是应用明确限流。",
|
||
"按模型查看 active-runs，处理异常长运行或评估上游模型真实配额后再调限制。",
|
||
)
|
||
elif max_elapsed >= 300:
|
||
_add_finding(
|
||
findings,
|
||
"warning",
|
||
"deerflow",
|
||
"long_active_run",
|
||
"存在持续很久的活动 Agent 运行",
|
||
[f"active={count}", f"max_elapsed={max_elapsed}s"],
|
||
"长 SSE/模型/工具任务会持续占用模型配额、连接和内存，但不一定异常。",
|
||
"在管理员并发页核对对应任务是否仍在产出；仅在确认卡死后中断。",
|
||
)
|
||
|
||
if not findings:
|
||
_add_finding(
|
||
findings,
|
||
"info",
|
||
"overall",
|
||
"no_decisive_signal",
|
||
"本次快照没有抓到决定性异常",
|
||
["没有命中脚本的高置信阈值"],
|
||
"间歇性卡顿恢复后采样只能说明恢复后的状态，不能排除曾发生过连接池、锁或资源尖峰。",
|
||
"在卡顿正在发生时重新运行，并启用 --deerflow-suite 与 --interference。",
|
||
)
|
||
findings.sort(key=lambda item: (SEVERITY_ORDER.get(str(item["severity"]), 99), str(item["area"]), str(item["code"])))
|
||
return findings
|
||
|
||
|
||
def build_quick_verdict(findings: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||
labels = {
|
||
"nginx": "Nginx/入口代理",
|
||
"mysql": "MySQL/数据库",
|
||
"deerflow": "DeerFlow 并发或热点路径",
|
||
"application": "接口自身或下游服务",
|
||
"resources": "服务器资源",
|
||
"network": "网络/DNS/TLS",
|
||
"overall": "暂未定位",
|
||
}
|
||
decisive = [item for item in findings if item.get("severity") in {"critical", "warning"}]
|
||
if not decisive:
|
||
return {
|
||
"headline": "本次没有抓到决定性异常；建议在卡顿正在发生时再次运行",
|
||
"areas": [],
|
||
"confidence": "low",
|
||
"evidence_titles": [str(item.get("title")) for item in findings[:3]],
|
||
}
|
||
areas: list[str] = []
|
||
for item in decisive:
|
||
label = labels.get(str(item.get("area")), str(item.get("area")))
|
||
if label not in areas:
|
||
areas.append(label)
|
||
if len(areas) >= 3:
|
||
break
|
||
primary_severity = str(decisive[0].get("severity"))
|
||
return {
|
||
"headline": "当前证据优先指向：" + "；同时关注：".join(areas),
|
||
"areas": areas,
|
||
"confidence": "high" if primary_severity == "critical" else "medium",
|
||
"evidence_titles": [str(item.get("title")) for item in decisive[:3]],
|
||
"note": "这是基于当前快照的定位方向，不是对根因的绝对证明。",
|
||
}
|
||
|
||
|
||
def _origin_for_request(url: str) -> str:
|
||
scheme, hostname, port = _origin_tuple(url)
|
||
try:
|
||
rendered_host = f"[{hostname}]" if ipaddress.ip_address(hostname).version == 6 else hostname
|
||
except ValueError:
|
||
rendered_host = hostname
|
||
default_port = 443 if scheme == "https" else 80
|
||
if port != default_port:
|
||
rendered_host = f"{rendered_host}:{port}"
|
||
return f"{scheme}://{rendered_host}"
|
||
|
||
|
||
def _report_output_path(raw: str | None) -> Path:
|
||
if raw:
|
||
requested = Path(raw).expanduser()
|
||
parent = requested.parent.resolve()
|
||
output = parent / requested.name
|
||
if output.is_symlink():
|
||
raise ValueError(f"refusing to replace symlink output path: {output}")
|
||
if output.exists() and not output.is_file():
|
||
raise ValueError(f"output path exists and is not a regular file: {output}")
|
||
return output
|
||
stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
||
return Path.cwd().resolve() / f"deerflow-latency-report-{stamp}.json"
|
||
|
||
|
||
def write_private_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
||
temporary_path = Path(temporary_name)
|
||
try:
|
||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||
handle.write("\n")
|
||
handle.flush()
|
||
os.fsync(handle.fileno())
|
||
with contextlib.suppress(OSError):
|
||
os.chmod(temporary_path, 0o600)
|
||
os.replace(temporary_path, path)
|
||
except Exception:
|
||
with contextlib.suppress(OSError):
|
||
os.close(descriptor)
|
||
with contextlib.suppress(OSError):
|
||
temporary_path.unlink()
|
||
raise
|
||
with contextlib.suppress(OSError):
|
||
os.chmod(path, 0o600)
|
||
|
||
|
||
def _format_ms(value: Any) -> str:
|
||
if value is None:
|
||
return "-"
|
||
try:
|
||
return f"{float(value):.1f}ms"
|
||
except (TypeError, ValueError):
|
||
return "-"
|
||
|
||
|
||
def print_console_summary(report: Mapping[str, Any], output_path: Path | None) -> None:
|
||
print("\n=== DeerFlow 接口卡顿诊断 ===")
|
||
print(f"时间: {report.get('started_at')}")
|
||
print("说明: HTTP/1.1 新连接测量；请求体、响应体、Token、Cookie 和查询参数值未写入报告。")
|
||
print("注意: 报告仍含路径、IP、容器名等基础设施元数据，请勿公开发布。")
|
||
warnings = report.get("security", {}).get("warnings", [])
|
||
if warnings:
|
||
print("安全警告:")
|
||
for warning in warnings:
|
||
print(f" - {warning}")
|
||
print("\n接口测量:")
|
||
for target in report.get("http", {}).get("targets", []):
|
||
summary = target.get("summary", {})
|
||
timings = summary.get("timings_ms", {})
|
||
print(
|
||
f" [{target.get('label')}] {target.get('url')} "
|
||
f"成功 {summary.get('successful_transport')}/{summary.get('requests')} "
|
||
f"总耗时p50={_format_ms(timings.get('total', {}).get('p50'))} "
|
||
f"等待首响应p50={_format_ms(timings.get('response_wait', {}).get('p50'))} "
|
||
f"DNS/TCP/TLS={_format_ms(timings.get('dns', {}).get('p50'))}/"
|
||
f"{_format_ms(timings.get('tcp_connect', {}).get('p50'))}/"
|
||
f"{_format_ms(timings.get('tls_handshake', {}).get('p50'))} "
|
||
f"状态={summary.get('status_counts')}"
|
||
)
|
||
|
||
host = report.get("server", {}).get("host", {})
|
||
if host.get("supported"):
|
||
print("\n服务器快照:")
|
||
print(f" CPU核数={host.get('cpu_count')} busy={host.get('cpu', {}).get('busy_pct')}% iowait={host.get('cpu', {}).get('iowait_pct')}% load={host.get('load_average')}")
|
||
print(f" 可用内存={host.get('memory', {}).get('available_pct')}% Swap使用={host.get('memory', {}).get('swap_used_pct')}% 最忙磁盘={host.get('disk_io', {}).get('busiest_device')} util={host.get('disk_io', {}).get('max_util_pct')}%")
|
||
|
||
verdict = report.get("quick_verdict", {})
|
||
if verdict:
|
||
print("\n>>> 一键判断:")
|
||
print(f" {verdict.get('headline')}")
|
||
for title in verdict.get("evidence_titles", []):
|
||
print(f" - {title}")
|
||
if verdict.get("note"):
|
||
print(f" 说明: {verdict.get('note')}")
|
||
|
||
print("\n优先结论:")
|
||
for index, finding in enumerate(report.get("findings", []), start=1):
|
||
severity = {"critical": "严重", "warning": "警告", "info": "提示", "ok": "正常"}.get(str(finding.get("severity")), str(finding.get("severity")))
|
||
print(f" {index}. [{severity}/{finding.get('area')}] {finding.get('title')}")
|
||
for evidence in finding.get("evidence", []):
|
||
print(f" 证据: {evidence}")
|
||
print(f" 判断: {finding.get('interpretation')}")
|
||
print(f" 下一步: {finding.get('next_step')}")
|
||
|
||
missing = report.get("coverage", {}).get("unavailable", [])
|
||
if missing:
|
||
print("\n未完成检查:")
|
||
for item in missing:
|
||
print(f" - {item}")
|
||
if output_path:
|
||
print(f"\n凭据脱敏 JSON 报告: {output_path}")
|
||
|
||
|
||
def _normalize_easy_url(raw: str) -> str:
|
||
value = raw.strip()
|
||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
||
value = value[1:-1].strip()
|
||
if not value:
|
||
raise ValueError("接口地址不能为空")
|
||
if "://" not in value:
|
||
value = "https://" + value
|
||
return value
|
||
|
||
|
||
def _option_present(arguments: Sequence[str], *names: str) -> bool:
|
||
return any(argument in names or any(argument.startswith(name + "=") for name in names) for argument in arguments)
|
||
|
||
|
||
def _verified_local_gateway_origin() -> str | None:
|
||
"""Return a loopback Gateway origin only after checking its public identity."""
|
||
ports: list[int] = [8001]
|
||
config_path = find_config_path(None)
|
||
if config_path:
|
||
config = read_safe_config(config_path)
|
||
configured_port = config.get("gateway.port")
|
||
if isinstance(configured_port, int) and 1 <= configured_port <= 65535:
|
||
ports.insert(0, configured_port)
|
||
for port in dict.fromkeys(ports):
|
||
origin = f"http://127.0.0.1:{port}"
|
||
request = urllib.request.Request(
|
||
origin + "/api/health",
|
||
method="GET",
|
||
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
|
||
)
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirectHandler())
|
||
try:
|
||
with opener.open(request, timeout=1.5) as response:
|
||
if response.status != 200:
|
||
continue
|
||
payload = json.loads(response.read(4096).decode("utf-8"))
|
||
except (OSError, TimeoutError, UnicodeError, ValueError, urllib.error.URLError, urllib.error.HTTPError):
|
||
continue
|
||
if isinstance(payload, dict) and payload.get("status") == "healthy" and payload.get("service") == "deer-flow-gateway":
|
||
return origin
|
||
return None
|
||
|
||
|
||
def _easy_optional_environment_args() -> list[str]:
|
||
arguments: list[str] = []
|
||
config_path = os.environ.get("DEERFLOW_DIAG_CONFIG")
|
||
mysql_file = os.environ.get("DEERFLOW_DIAG_MYSQL_CNF")
|
||
mysql_url = os.environ.get("DEERFLOW_DIAG_MYSQL_URL")
|
||
nginx_status = os.environ.get("DEERFLOW_DIAG_NGINX_STATUS_URL")
|
||
admin_token = os.environ.get("DEERFLOW_DIAG_ADMIN_TOKEN")
|
||
if config_path:
|
||
arguments.extend(["--config", config_path])
|
||
if mysql_file:
|
||
arguments.extend(["--mysql-defaults-file", mysql_file])
|
||
elif mysql_url:
|
||
arguments.extend(["--mysql", "--mysql-url-env", "DEERFLOW_DIAG_MYSQL_URL"])
|
||
if nginx_status:
|
||
arguments.extend(["--nginx-status-url", nginx_status])
|
||
if admin_token:
|
||
arguments.extend(["--admin-token-env", "DEERFLOW_DIAG_ADMIN_TOKEN"])
|
||
return arguments
|
||
|
||
|
||
def _easy_auth_from_environment() -> list[str]:
|
||
if os.environ.get("DEERFLOW_DIAG_TOKEN"):
|
||
return ["--bearer-env", "DEERFLOW_DIAG_TOKEN"]
|
||
if os.environ.get("DEERFLOW_DIAG_COOKIE"):
|
||
return ["--cookie-env", "DEERFLOW_DIAG_COOKIE"]
|
||
return []
|
||
|
||
|
||
def _store_easy_secret(kind: str, value: str) -> tuple[list[str], str | None]:
|
||
if not value:
|
||
return [], None
|
||
env_name = f"_DEERFLOW_DIAG_EASY_{kind.upper()}_{os.getpid()}"
|
||
os.environ[env_name] = value
|
||
option = "--bearer-env" if kind == "token" else "--cookie-env"
|
||
return [option, env_name], env_name
|
||
|
||
|
||
def _interactive_easy_arguments() -> tuple[list[str], list[str]]:
|
||
print("\n=== DeerFlow 一键卡顿诊断 ===")
|
||
print("只需选择接口类型并粘贴地址；不会修改、重启或替换后端。")
|
||
target_url = _normalize_easy_url(input("\n请粘贴卡顿接口 URL: "))
|
||
print("\n接口类型：")
|
||
print(" 1. 普通只读 GET / Skills（默认，深度检查，会请求目标 4 次）")
|
||
print(" 2. 登录 POST（只请求 1 次）")
|
||
print(" 3. Agent SSE POST（只请求 1 次，只等待首个响应实体字节）")
|
||
mode = input("请选择 [1/2/3，默认 1]: ").strip() or "1"
|
||
if mode not in {"1", "2", "3"}:
|
||
raise ValueError("接口类型只能选择 1、2 或 3")
|
||
|
||
arguments = [
|
||
"--easy-mode",
|
||
"--url",
|
||
target_url,
|
||
"--server-checks",
|
||
"--deerflow-suite",
|
||
"--concurrency",
|
||
"1",
|
||
"--connect-timeout",
|
||
"5",
|
||
]
|
||
cleanup_env_names: list[str] = []
|
||
if mode == "1":
|
||
arguments.extend(
|
||
[
|
||
"--count",
|
||
"3",
|
||
"--interference",
|
||
"--first-byte-timeout",
|
||
"30",
|
||
"--total-timeout",
|
||
"60",
|
||
]
|
||
)
|
||
else:
|
||
body_path = input("请求 JSON 文件路径: ").strip()
|
||
if not body_path:
|
||
raise ValueError("POST 诊断必须提供请求 JSON 文件")
|
||
resolved_body_path = Path(body_path).expanduser().resolve()
|
||
if not resolved_body_path.is_file():
|
||
raise ValueError(f"请求文件不存在: {resolved_body_path}")
|
||
arguments.extend(
|
||
[
|
||
"--method",
|
||
"POST",
|
||
"--header",
|
||
"Content-Type: application/json",
|
||
"--data-file",
|
||
str(resolved_body_path),
|
||
"--count",
|
||
"1",
|
||
]
|
||
)
|
||
if mode == "2":
|
||
arguments.extend(["--read-mode", "full", "--first-byte-timeout", "60", "--total-timeout", "90"])
|
||
else:
|
||
arguments.extend(
|
||
[
|
||
"--header",
|
||
"Accept: text/event-stream",
|
||
"--read-mode",
|
||
"first-byte",
|
||
"--first-byte-timeout",
|
||
"180",
|
||
"--total-timeout",
|
||
"240",
|
||
]
|
||
)
|
||
|
||
environment_auth = _easy_auth_from_environment()
|
||
if environment_auth:
|
||
env_name = environment_auth[-1]
|
||
print(f"检测到 {env_name}，将用于用户输入的目标地址；值不会写入报告。")
|
||
arguments.extend(environment_auth)
|
||
else:
|
||
default_auth = "1" if mode == "3" else "0"
|
||
auth_choice = input(f"鉴权方式 [0=无, 1=Bearer Token, 2=Cookie，默认 {default_auth}]: ").strip() or default_auth
|
||
if auth_choice not in {"0", "1", "2"}:
|
||
raise ValueError("鉴权方式只能选择 0、1 或 2")
|
||
if auth_choice == "1":
|
||
secret_args, env_name = _store_easy_secret("token", getpass.getpass("Bearer Token（输入不显示）: ").strip())
|
||
arguments.extend(secret_args)
|
||
if env_name:
|
||
cleanup_env_names.append(env_name)
|
||
elif auth_choice == "2":
|
||
secret_args, env_name = _store_easy_secret("cookie", getpass.getpass("Cookie 值（输入不显示，不含 Cookie: 前缀）: ").strip())
|
||
arguments.extend(secret_args)
|
||
if env_name:
|
||
cleanup_env_names.append(env_name)
|
||
|
||
local_origin = _verified_local_gateway_origin()
|
||
if local_origin:
|
||
arguments.extend(["--easy-local-origin", local_origin])
|
||
print(f"已确认本机 DeerFlow Gateway：{local_origin}；仅对比公开 health，不会向它复制目标查询参数、凭据或请求体。")
|
||
on_server = input("当前脚本是否运行在这个 DeerFlow 的实际部署服务器上？[Y/n]: ").strip().lower()
|
||
if on_server in {"", "y", "yes", "是"}:
|
||
arguments.append("--easy-target-host-confirmed")
|
||
elif on_server not in {"n", "no", "否"}:
|
||
raise ValueError("请回答 y 或 n")
|
||
arguments.extend(_easy_optional_environment_args())
|
||
return arguments, cleanup_env_names
|
||
|
||
|
||
def _positional_easy_arguments(raw_arguments: Sequence[str]) -> list[str]:
|
||
target_url = _normalize_easy_url(str(raw_arguments[0]))
|
||
rest = list(raw_arguments[1:])
|
||
arguments = [
|
||
"--easy-mode",
|
||
"--url",
|
||
target_url,
|
||
"--count",
|
||
"1",
|
||
"--concurrency",
|
||
"1",
|
||
"--deerflow-suite",
|
||
"--server-checks",
|
||
"--connect-timeout",
|
||
"5",
|
||
"--first-byte-timeout",
|
||
"30",
|
||
"--total-timeout",
|
||
"60",
|
||
]
|
||
if not _option_present(rest, "--bearer-env", "--cookie-env", "--header-env", "--header"):
|
||
arguments.extend(_easy_auth_from_environment())
|
||
local_origin = _verified_local_gateway_origin()
|
||
if local_origin:
|
||
arguments.extend(["--easy-local-origin", local_origin])
|
||
if os.environ.get("DEERFLOW_DIAG_ON_SERVER", "").strip().lower() in {"1", "true", "yes", "y"}:
|
||
arguments.append("--easy-target-host-confirmed")
|
||
arguments.extend(_easy_optional_environment_args())
|
||
arguments.extend(rest)
|
||
return arguments
|
||
|
||
|
||
def prepare_cli_arguments(raw_arguments: Sequence[str]) -> tuple[list[str], list[str]]:
|
||
arguments = list(raw_arguments)
|
||
if not arguments:
|
||
if not sys.stdin.isatty():
|
||
raise ValueError("无参数模式需要交互终端；非交互执行请使用: python3 diagnose_deerflow_latency.py 'https://地址'")
|
||
return _interactive_easy_arguments()
|
||
if not arguments[0].startswith("-"):
|
||
return _positional_easy_arguments(arguments), []
|
||
return arguments, []
|
||
|
||
|
||
def build_argument_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description=("Diagnose DeerFlow latency across HTTP/Nginx/database/concurrency/host layers. Run it while the slowdown is happening for the strongest evidence."))
|
||
parser.add_argument("--url", action="append", default=[], help="Target URL; may be repeated")
|
||
parser.add_argument(
|
||
"--url-env",
|
||
action="append",
|
||
default=[],
|
||
metavar="ENV_NAME",
|
||
help="Read a target URL from an environment variable; may be repeated",
|
||
)
|
||
parser.add_argument(
|
||
"--compare-url",
|
||
"--direct-url",
|
||
action="append",
|
||
default=[],
|
||
dest="compare_url",
|
||
help="Same request through another path, e.g. http://127.0.0.1:8001/...",
|
||
)
|
||
parser.add_argument(
|
||
"--allow-cross-origin-compare",
|
||
action="store_true",
|
||
help="Allow credentials/request body to reach an explicitly supplied comparison origin",
|
||
)
|
||
parser.add_argument("--method", default="GET", help="HTTP method (default: GET)")
|
||
parser.add_argument(
|
||
"--query-env",
|
||
action="append",
|
||
default=[],
|
||
metavar="KEY=ENV_NAME",
|
||
help="Append/replace a query value from an environment variable on every target URL",
|
||
)
|
||
parser.add_argument("--header", action="append", default=[], help="Request header NAME:VALUE; secrets should use --header-env")
|
||
parser.add_argument(
|
||
"--header-env",
|
||
action="append",
|
||
default=[],
|
||
metavar="HEADER=ENV_NAME",
|
||
help="Read a header value from an environment variable",
|
||
)
|
||
parser.add_argument("--bearer-env", help="Environment variable containing a Bearer token")
|
||
parser.add_argument("--cookie-env", help="Environment variable containing a Cookie header value")
|
||
parser.add_argument("--data-file", help="Request body file; '-' reads stdin. Body content is never reported")
|
||
parser.add_argument("--count", type=int, help="Total fresh-connection samples (safe methods default 5; unsafe default 1)")
|
||
parser.add_argument("--concurrency", type=int, default=1, help="Concurrent samples; keep low on production (default 1)")
|
||
parser.add_argument("--interval", type=float, default=0.2, help="Seconds between sequential samples")
|
||
parser.add_argument(
|
||
"--allow-unsafe-replay",
|
||
action="store_true",
|
||
help="Explicitly allow repeating/concurrently sending POST/PUT/PATCH/DELETE requests",
|
||
)
|
||
parser.add_argument(
|
||
"--read-mode",
|
||
choices=("auto", "headers", "first-byte", "full"),
|
||
default="auto",
|
||
help="auto uses first-byte for SSE and full for other responses",
|
||
)
|
||
parser.add_argument("--connect-timeout", type=float, default=5.0)
|
||
parser.add_argument("--first-byte-timeout", type=float, default=30.0)
|
||
parser.add_argument("--read-timeout", type=float, default=5.0)
|
||
parser.add_argument("--total-timeout", type=float, default=120.0, help="Hard deadline per HTTP request")
|
||
parser.add_argument("--max-response-bytes", type=int, default=1024 * 1024)
|
||
parser.add_argument("--follow-redirects", action="store_true", help="Follow at most --max-redirects redirects")
|
||
parser.add_argument("--max-redirects", type=int, default=3)
|
||
parser.add_argument("--proxy", help="Explicit http:// forward proxy; credentials are never reported")
|
||
parser.add_argument("--proxy-env", help="Environment variable holding an explicit http:// proxy URL")
|
||
parser.add_argument("--use-env-proxy", action="store_true", help="Use HTTP(S)_PROXY/NO_PROXY environment settings")
|
||
parser.add_argument("--ip-family", choices=("auto", "ipv4", "ipv6"), default="auto")
|
||
parser.add_argument("--ca-file", help="Custom CA bundle for HTTPS")
|
||
parser.add_argument("--insecure", action="store_true", help="Disable TLS certificate verification (recorded as unsafe)")
|
||
parser.add_argument(
|
||
"--allow-insecure-http",
|
||
action="store_true",
|
||
help="Allow credentials, environment-backed query values, or a request body over non-loopback plaintext HTTP",
|
||
)
|
||
parser.add_argument(
|
||
"--health-baselines",
|
||
action=argparse.BooleanOptionalAction,
|
||
default=True,
|
||
help="Probe /health, /api/health and /api/langgraph/health once per origin",
|
||
)
|
||
parser.add_argument(
|
||
"--deerflow-suite",
|
||
action="store_true",
|
||
help="Probe safe DeerFlow baselines (auth/options and, with auth, me/agents/skills)",
|
||
)
|
||
parser.add_argument(
|
||
"--interference",
|
||
action="store_true",
|
||
help="Send one extra target request while repeatedly probing /api/health",
|
||
)
|
||
parser.add_argument("--interference-url", help="Custom sentinel URL (default: target origin /api/health)")
|
||
parser.add_argument("--sentinel-interval", type=float, default=0.25)
|
||
parser.add_argument("--max-sentinel-probes", type=int, default=120)
|
||
parser.add_argument(
|
||
"--server-checks",
|
||
action=argparse.BooleanOptionalAction,
|
||
default=None,
|
||
help="Collect Linux/Nginx/Docker/log/database checks (auto-enabled on Linux)",
|
||
)
|
||
parser.add_argument("--resource-sample-seconds", type=float, default=1.0)
|
||
parser.add_argument("--config", help="DeerFlow config.yaml path")
|
||
parser.add_argument("--log", action="append", default=[], help="Additional log file to scan; may be repeated")
|
||
parser.add_argument("--log-tail-bytes", type=int, default=4 * 1024 * 1024)
|
||
parser.add_argument(
|
||
"--mysql",
|
||
action=argparse.BooleanOptionalAction,
|
||
default=None,
|
||
help="Run read-only MySQL checks (requires this flag, --mysql-defaults-file, or --mysql-host)",
|
||
)
|
||
parser.add_argument("--mysql-url-env", default="MYSQL_DATABASE_URL", help="Environment variable holding MySQL URL")
|
||
parser.add_argument("--mysql-defaults-file", help="Existing mysql client option file (recommended alternative to URL env)")
|
||
parser.add_argument("--mysql-host", help="Host-only TCP test when credentials are unavailable")
|
||
parser.add_argument("--mysql-port", type=int, default=3306)
|
||
parser.add_argument("--mysql-probes", type=int, default=3)
|
||
parser.add_argument("--nginx-status-url", help="Optional Nginx stub_status URL")
|
||
parser.add_argument("--admin-origin", help="Gateway origin for admin read-only concurrency APIs")
|
||
parser.add_argument("--admin-token-env", help="Environment variable holding an admin JWT")
|
||
parser.add_argument(
|
||
"--allow-cross-origin-admin",
|
||
action="store_true",
|
||
help="Allow the admin JWT to be sent to an explicitly supplied origin different from the primary target",
|
||
)
|
||
parser.add_argument("--output", help="JSON report path (default: ./deerflow-latency-report-<time>.json)")
|
||
parser.add_argument("--no-output", action="store_true", help="Do not write a JSON report")
|
||
parser.add_argument("--json-stdout", action="store_true", help="Print the full JSON report instead of the Chinese summary")
|
||
parser.add_argument("--easy-mode", action="store_true", help=argparse.SUPPRESS)
|
||
parser.add_argument("--easy-local-origin", help=argparse.SUPPRESS)
|
||
parser.add_argument("--easy-target-host-confirmed", action="store_true", help=argparse.SUPPRESS)
|
||
return parser
|
||
|
||
|
||
def _apply_query_env(url: str, specifications: Sequence[str]) -> str:
|
||
parsed = urllib.parse.urlsplit(url)
|
||
pairs = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
|
||
for specification in specifications:
|
||
key, separator, env_name = specification.partition("=")
|
||
key = key.strip()
|
||
env_name = env_name.strip()
|
||
if not separator or not re.fullmatch(r"[A-Za-z0-9_.~-]{1,64}", key) or not env_name:
|
||
raise ValueError("--query-env must use a safe KEY=ENV_NAME")
|
||
value = os.environ.get(env_name)
|
||
if value is None:
|
||
raise ValueError(f"environment variable {env_name} is not set")
|
||
pairs = [(old_key, old_value) for old_key, old_value in pairs if old_key != key]
|
||
pairs.append((key, value))
|
||
query = urllib.parse.urlencode(pairs)
|
||
return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, parsed.fragment))
|
||
|
||
|
||
def _load_request_headers(args: argparse.Namespace) -> tuple[dict[str, str], list[str]]:
|
||
headers: dict[str, str] = {}
|
||
names_by_lower: dict[str, str] = {}
|
||
warnings: list[str] = []
|
||
|
||
def add_header(name: str, value: str) -> None:
|
||
lowered = name.lower()
|
||
if lowered in names_by_lower:
|
||
raise ValueError(f"duplicate request header names differ only by case: {names_by_lower[lowered]} and {name}")
|
||
names_by_lower[lowered] = name
|
||
headers[name] = value
|
||
|
||
for raw in args.header:
|
||
name, value = parse_header_argument(raw)
|
||
add_header(name, value)
|
||
if _header_is_secret(name):
|
||
warnings.append(f"Secret-like header {name} was supplied on the command line; use --header-env next time")
|
||
for raw in args.header_env:
|
||
name, value = parse_header_env_argument(raw)
|
||
add_header(name, value)
|
||
if args.bearer_env:
|
||
token = os.environ.get(args.bearer_env)
|
||
if token is None:
|
||
raise ValueError(f"Environment variable {args.bearer_env} is not set")
|
||
add_header("Authorization", token if token.lower().startswith("bearer ") else f"Bearer {token}")
|
||
if args.cookie_env:
|
||
cookie = os.environ.get(args.cookie_env)
|
||
if cookie is None:
|
||
raise ValueError(f"Environment variable {args.cookie_env} is not set")
|
||
add_header("Cookie", cookie)
|
||
return headers, warnings
|
||
|
||
|
||
def _load_request_body(path: str | None) -> bytes:
|
||
if not path:
|
||
return b""
|
||
limit = 16 * 1024 * 1024
|
||
if path == "-":
|
||
data = sys.stdin.buffer.read(limit + 1)
|
||
else:
|
||
body_path = Path(path).expanduser()
|
||
with body_path.open("rb") as handle:
|
||
data = handle.read(limit + 1)
|
||
if len(data) > limit:
|
||
raise ValueError("Request body exceeds the 16 MiB diagnostic safety limit")
|
||
return data
|
||
|
||
|
||
def _unavailable_coverage(report: Mapping[str, Any]) -> list[str]:
|
||
unavailable: list[str] = []
|
||
server = report.get("server", {})
|
||
if server:
|
||
for key, label in (
|
||
("host", "Linux /proc resource snapshot"),
|
||
("sockets", "socket/listen queue snapshot"),
|
||
("docker", "Docker snapshot"),
|
||
("nginx", "Nginx effective configuration"),
|
||
("mysql", "MySQL authenticated diagnostics"),
|
||
):
|
||
entry = server.get(key, {})
|
||
explicitly_unavailable = entry.get("available") is False or (key == "host" and entry.get("supported") is False)
|
||
if entry and explicitly_unavailable:
|
||
unavailable.append(f"{label}: {entry.get('reason') or entry.get('error') or 'unavailable'}")
|
||
admin = report.get("deerflow", {}).get("admin_concurrency", {})
|
||
if admin.get("enabled") and admin.get("available") is False:
|
||
unavailable.append(f"DeerFlow admin concurrency APIs: {admin.get('reason', 'unavailable')}")
|
||
elif admin.get("partial"):
|
||
unavailable.append("DeerFlow admin concurrency APIs (partial): " + ", ".join(str(name) for name in admin.get("failed_endpoints", [])))
|
||
if report.get("request", {}).get("easy_mode"):
|
||
mysql = server.get("mysql", {})
|
||
if mysql.get("enabled") is False:
|
||
unavailable.append("MySQL authenticated diagnostics: 未提供 DEERFLOW_DIAG_MYSQL_CNF/DEERFLOW_DIAG_MYSQL_URL，未执行认证检查")
|
||
if admin.get("enabled") is False:
|
||
unavailable.append("DeerFlow admin concurrency APIs: 未提供 DEERFLOW_DIAG_ADMIN_TOKEN，未核验活动运行数和模型并发上限")
|
||
if server.get("scope", {}).get("target_host_confirmed") is False:
|
||
unavailable.append("Server resource attribution: 未确认脚本执行机就是目标 DeerFlow 服务器")
|
||
return unavailable
|
||
|
||
|
||
def main(argv: Sequence[str] | None = None) -> int:
|
||
parser = build_argument_parser()
|
||
args = parser.parse_args(argv)
|
||
try:
|
||
if args.proxy and args.proxy_env:
|
||
raise ValueError("use only one of --proxy and --proxy-env")
|
||
if args.proxy_env:
|
||
args.proxy = os.environ.get(args.proxy_env)
|
||
if not args.proxy:
|
||
raise ValueError(f"environment variable {args.proxy_env} is not set")
|
||
method = args.method.upper().strip()
|
||
if not re.fullmatch(r"[A-Z]+", method):
|
||
raise ValueError("HTTP method must contain only letters")
|
||
target_urls = list(args.url)
|
||
for env_name in args.url_env:
|
||
env_url = os.environ.get(env_name)
|
||
if not env_url:
|
||
raise ValueError(f"environment variable {env_name} is not set")
|
||
target_urls.append(env_url)
|
||
if not target_urls:
|
||
raise ValueError("at least one --url or --url-env is required")
|
||
urls = [_apply_query_env(url, args.query_env) for url in target_urls + list(args.compare_url)]
|
||
for url in urls:
|
||
parsed_url = urllib.parse.urlsplit(url)
|
||
_origin_tuple(url)
|
||
if parsed_url.username is not None or parsed_url.password is not None:
|
||
raise ValueError("target URL userinfo is not allowed; pass credentials through an environment-backed header")
|
||
if any(ord(character) < 32 or ord(character) == 127 for character in url):
|
||
raise ValueError("target URL contains a control character")
|
||
if args.proxy:
|
||
parsed_proxy = urllib.parse.urlsplit(args.proxy)
|
||
if parsed_proxy.scheme.lower() != "http" or not parsed_proxy.hostname:
|
||
raise ValueError("--proxy/--proxy-env must contain a valid http:// proxy URL")
|
||
_ = parsed_proxy.port
|
||
for label, auxiliary_url in (
|
||
("--admin-origin", args.admin_origin),
|
||
("--interference-url", args.interference_url),
|
||
("--nginx-status-url", args.nginx_status_url),
|
||
("--easy-local-origin", args.easy_local_origin),
|
||
):
|
||
if not auxiliary_url:
|
||
continue
|
||
parsed_auxiliary = urllib.parse.urlsplit(auxiliary_url)
|
||
_origin_tuple(auxiliary_url)
|
||
if parsed_auxiliary.username is not None or parsed_auxiliary.password is not None:
|
||
raise ValueError(f"{label} must not contain URL userinfo")
|
||
if any(ord(character) < 32 or ord(character) == 127 for character in auxiliary_url):
|
||
raise ValueError(f"{label} contains a control character")
|
||
if args.easy_local_origin:
|
||
easy_scheme, easy_host, _easy_port = _origin_tuple(args.easy_local_origin)
|
||
if easy_scheme != "http" or not _host_is_loopback(easy_host):
|
||
raise ValueError("--easy-local-origin must be a loopback http:// origin")
|
||
if args.ca_file:
|
||
resolved_ca_file = Path(args.ca_file).expanduser().resolve()
|
||
if not resolved_ca_file.is_file():
|
||
raise ValueError(f"CA file not found: {args.ca_file}")
|
||
args.ca_file = str(resolved_ca_file)
|
||
headers, input_warnings = _load_request_headers(args)
|
||
for raw_url in list(args.url) + list(args.compare_url):
|
||
for query_key, query_value in urllib.parse.parse_qsl(
|
||
urllib.parse.urlsplit(raw_url).query,
|
||
keep_blank_values=True,
|
||
):
|
||
if query_value and SECRET_QUERY_RE.search(query_key):
|
||
input_warnings.append(f"Secret-like query parameter {query_key} was supplied on the command line; use --query-env")
|
||
if args.proxy and urllib.parse.urlsplit(args.proxy).username is not None and not args.proxy_env:
|
||
input_warnings.append("Proxy credentials were supplied on the command line; use --proxy-env to keep them out of shell history")
|
||
proxy_has_credentials = bool(args.proxy and (urllib.parse.urlsplit(args.proxy).username is not None or any(name.lower() == "proxy-authorization" for name in headers)))
|
||
if proxy_has_credentials:
|
||
input_warnings.append("The supported proxy scheme is http://; proxy authentication is exposed to the proxy link before any target TLS tunnel")
|
||
if args.allow_cross_origin_compare:
|
||
input_warnings.append("Cross-origin comparison was enabled; supplied headers, environment-backed query values, and request body may be sent to comparison hosts")
|
||
body = _load_request_body(args.data_file)
|
||
primary_origin = _origin_tuple(urls[0])
|
||
if args.admin_token_env and args.admin_origin:
|
||
admin_origin_tuple = _origin_tuple(args.admin_origin)
|
||
if admin_origin_tuple != primary_origin and not args.allow_cross_origin_admin:
|
||
raise ValueError("--admin-origin differs from the primary target and would receive an admin JWT; review it and add --allow-cross-origin-admin")
|
||
if admin_origin_tuple != primary_origin:
|
||
input_warnings.append("Cross-origin admin diagnostics were enabled; the admin JWT will be sent to the supplied admin origin")
|
||
sensitive_request = bool(body) or bool(args.query_env) or any(_header_is_secret(name) for name in headers) or any(_url_has_secret_query(url) for url in urls)
|
||
plaintext_sensitive_origins: set[str] = set()
|
||
proxy_selection_options = ProbeOptions(
|
||
proxy=args.proxy,
|
||
use_env_proxy=args.use_env_proxy,
|
||
)
|
||
if sensitive_request:
|
||
for url in urls:
|
||
scheme, host, _port = _origin_tuple(url)
|
||
if scheme != "http":
|
||
continue
|
||
selected_proxy = _select_proxy(url, proxy_selection_options)
|
||
if not _host_is_loopback(host) or selected_proxy:
|
||
plaintext_sensitive_origins.add(sanitize_origin(url))
|
||
admin_origin_for_transport = args.admin_origin or _origin_for_request(urls[0])
|
||
admin_token_present = bool(args.admin_token_env and os.environ.get(args.admin_token_env))
|
||
admin_scheme, admin_host, _admin_port = _origin_tuple(admin_origin_for_transport)
|
||
if admin_token_present and admin_scheme == "http" and not _host_is_loopback(admin_host):
|
||
plaintext_sensitive_origins.add(sanitize_origin(admin_origin_for_transport))
|
||
if plaintext_sensitive_origins and not args.allow_insecure_http:
|
||
rendered_origins = ", ".join(sorted(plaintext_sensitive_origins))
|
||
raise ValueError(f"credentials/environment-backed query values/request body would be sent over non-loopback plaintext HTTP to {rendered_origins}; use HTTPS or review the risk and add --allow-insecure-http")
|
||
if plaintext_sensitive_origins:
|
||
input_warnings.append("Plaintext HTTP was explicitly allowed for sensitive request material: " + ", ".join(sorted(plaintext_sensitive_origins)))
|
||
if not args.allow_cross_origin_compare:
|
||
primary_has_query = bool(urllib.parse.urlsplit(urls[0]).query)
|
||
for compare_url in urls[1:]:
|
||
compare_has_query = bool(urllib.parse.urlsplit(compare_url).query)
|
||
if _origin_tuple(compare_url) != primary_origin and (
|
||
headers
|
||
or body
|
||
or args.query_env
|
||
or primary_has_query
|
||
or compare_has_query
|
||
or method not in SAFE_METHODS
|
||
):
|
||
raise ValueError("a comparison URL has a different origin and would receive headers/body/query values; review it and add --allow-cross-origin-compare")
|
||
count = args.count if args.count is not None else (5 if method in SAFE_METHODS else 1)
|
||
if count < 1 or count > 1000:
|
||
raise ValueError("--count must be between 1 and 1000")
|
||
if args.concurrency < 1 or args.concurrency > 100:
|
||
raise ValueError("--concurrency must be between 1 and 100")
|
||
for name, value, minimum, maximum in (
|
||
("--connect-timeout", args.connect_timeout, 0.1, 300.0),
|
||
("--first-byte-timeout", args.first_byte_timeout, 0.1, 3600.0),
|
||
("--read-timeout", args.read_timeout, 0.1, 3600.0),
|
||
("--total-timeout", args.total_timeout, 0.1, 7200.0),
|
||
("--interval", args.interval, 0.0, 3600.0),
|
||
("--sentinel-interval", args.sentinel_interval, 0.05, 60.0),
|
||
("--resource-sample-seconds", args.resource_sample_seconds, 0.1, 30.0),
|
||
):
|
||
if not math.isfinite(value) or value < minimum or value > maximum:
|
||
raise ValueError(f"{name} must be a finite number between {minimum} and {maximum}")
|
||
if args.mysql_port < 1 or args.mysql_port > 65535:
|
||
raise ValueError("--mysql-port must be between 1 and 65535")
|
||
replay_url_count = len(urls) + (1 if args.interference else 0)
|
||
validate_replay_safety(
|
||
method=method,
|
||
url_count=replay_url_count,
|
||
count=count,
|
||
concurrency=args.concurrency,
|
||
follow_redirects=args.follow_redirects,
|
||
allow_unsafe_replay=args.allow_unsafe_replay,
|
||
)
|
||
if args.max_response_bytes < 1 or args.max_response_bytes > 64 * 1024 * 1024:
|
||
raise ValueError("--max-response-bytes must be between 1 and 67108864")
|
||
if args.interference and args.max_sentinel_probes < 1:
|
||
raise ValueError("--max-sentinel-probes must be positive")
|
||
planned_output_path = None if args.no_output else _report_output_path(args.output)
|
||
except (OSError, ValueError) as exc:
|
||
parser.error(str(exc))
|
||
|
||
options = ProbeOptions(
|
||
method=method,
|
||
headers=headers,
|
||
body=body,
|
||
connect_timeout=max(0.1, args.connect_timeout),
|
||
first_byte_timeout=max(0.1, args.first_byte_timeout),
|
||
read_timeout=max(0.1, args.read_timeout),
|
||
total_timeout=max(0.1, args.total_timeout),
|
||
max_response_bytes=args.max_response_bytes,
|
||
read_mode=args.read_mode,
|
||
insecure=args.insecure,
|
||
ca_file=args.ca_file,
|
||
follow_redirects=args.follow_redirects,
|
||
max_redirects=max(0, min(args.max_redirects, 10)),
|
||
proxy=args.proxy,
|
||
use_env_proxy=args.use_env_proxy,
|
||
ip_family=args.ip_family,
|
||
)
|
||
|
||
config_path = find_config_path(args.config)
|
||
config = read_safe_config(config_path) if config_path else {"found": False}
|
||
system_settings = read_safe_system_settings(config_path)
|
||
started_at = utc_now_iso()
|
||
report: dict[str, Any] = {
|
||
"report_version": REPORT_VERSION,
|
||
"started_at": started_at,
|
||
"finished_at": None,
|
||
"request": {
|
||
"method": method,
|
||
"easy_mode": bool(args.easy_mode),
|
||
"target_count": len(urls),
|
||
"sample_count_per_target": count,
|
||
"concurrency": args.concurrency,
|
||
"read_mode": args.read_mode,
|
||
"body_bytes": len(body),
|
||
"header_names": sorted(headers),
|
||
"tls_verification": not args.insecure,
|
||
"proxy_mode": "explicit" if args.proxy else ("environment" if args.use_env_proxy else "direct"),
|
||
},
|
||
"security": {
|
||
"request_body_saved": False,
|
||
"response_body_saved": False,
|
||
"secret_header_values_saved": False,
|
||
"url_query_values_saved": False,
|
||
"warnings": input_warnings
|
||
+ (
|
||
["TLS certificate verification was disabled for one or more HTTPS probes"]
|
||
if args.insecure and any(urllib.parse.urlsplit(candidate).scheme.lower() == "https" for candidate in urls + [value for value in (args.admin_origin, args.interference_url, args.nginx_status_url) if value])
|
||
else []
|
||
),
|
||
},
|
||
"deerflow": {"config": config, "system_settings": system_settings},
|
||
"http": {"targets": []},
|
||
"server": {},
|
||
"findings": [],
|
||
"coverage": {"unavailable": []},
|
||
"caveats": [
|
||
"Fresh HTTP/1.1 connections are used; browser HTTP/2 multiplexing and keepalive reuse are not reproduced.",
|
||
"A single snapshot cannot disprove an intermittent problem; rerun while the slowdown is happening.",
|
||
"High response_wait localizes the delay after upload and before first response, but does not alone prove DB, model, Nginx, or disk root cause.",
|
||
"Health-sentinel evidence can be hidden by multiple workers or load-balancer routing.",
|
||
"The report removes credential values but contains infrastructure metadata such as paths, IPs, container names, database hosts and model names; do not publish it publicly.",
|
||
],
|
||
}
|
||
|
||
server_checks = args.server_checks if args.server_checks is not None else platform.system() == "Linux"
|
||
resource_executor: concurrent.futures.ThreadPoolExecutor | None = None
|
||
resource_future: concurrent.futures.Future[dict[str, Any]] | None = None
|
||
if server_checks:
|
||
resource_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||
resource_future = resource_executor.submit(collect_linux_snapshot, args.resource_sample_seconds)
|
||
|
||
try:
|
||
if args.easy_mode and not args.json_stdout:
|
||
print("\n[1/5] 正在测量目标接口……", flush=True)
|
||
for index, url in enumerate(urls):
|
||
label = "target" if index == 0 else f"compare-{index}"
|
||
results, wall_ms = run_probe_batch(
|
||
url,
|
||
options,
|
||
count=count,
|
||
concurrency=args.concurrency,
|
||
interval=max(0.0, args.interval),
|
||
)
|
||
report["http"]["targets"].append(
|
||
{
|
||
"label": label,
|
||
"url": sanitize_url(url),
|
||
"wall_ms": wall_ms,
|
||
"observed_rate_including_interval_requests_per_sec": round(
|
||
count / max(wall_ms / 1000.0, 0.001),
|
||
3,
|
||
),
|
||
"summary": summarize_probe_results(results),
|
||
"samples": results,
|
||
}
|
||
)
|
||
|
||
origins = [_origin_for_request(url) for url in urls]
|
||
if args.easy_mode and not args.json_stdout:
|
||
print("[2/5] 正在检查公网 health 与本机 Gateway……", flush=True)
|
||
if args.health_baselines:
|
||
report["http"]["health_baselines"] = run_health_baselines(origins, options)
|
||
if args.easy_local_origin:
|
||
local_health_options = ProbeOptions(
|
||
method="GET",
|
||
headers={},
|
||
connect_timeout=min(options.connect_timeout, 3.0),
|
||
first_byte_timeout=min(options.first_byte_timeout, 5.0),
|
||
read_timeout=min(options.read_timeout, 2.0),
|
||
total_timeout=min(options.total_timeout, 10.0),
|
||
max_response_bytes=65536,
|
||
read_mode="full",
|
||
insecure=False,
|
||
follow_redirects=False,
|
||
proxy=None,
|
||
use_env_proxy=False,
|
||
ip_family="ipv4",
|
||
)
|
||
report["http"]["local_gateway_health"] = probe_url(
|
||
urllib.parse.urljoin(args.easy_local_origin.rstrip("/") + "/", "api/health"),
|
||
local_health_options,
|
||
)
|
||
if args.easy_mode and not args.json_stdout:
|
||
print("[3/5] 正在检查 DeerFlow 基线与请求排队迹象……", flush=True)
|
||
if args.deerflow_suite:
|
||
authenticated = any(_header_is_secret(name) for name in headers)
|
||
report["http"]["deerflow_suite"] = run_deerflow_suite(origins[0], options, authenticated=authenticated)
|
||
if args.interference:
|
||
sentinel_url = args.interference_url or urllib.parse.urljoin(origins[0] + "/", "api/health")
|
||
report["http"]["interference"] = run_interference_probe(
|
||
urls[0],
|
||
sentinel_url,
|
||
options,
|
||
interval=max(0.1, args.sentinel_interval),
|
||
max_sentinel_probes=min(max(args.max_sentinel_probes, 1), 1000),
|
||
)
|
||
|
||
admin_origin = args.admin_origin or origins[0]
|
||
report["deerflow"]["admin_concurrency"] = collect_admin_concurrency(
|
||
admin_origin,
|
||
token_env=args.admin_token_env,
|
||
timeout=min(max(args.first_byte_timeout, 1.0), 30.0),
|
||
insecure=args.insecure,
|
||
ca_file=args.ca_file,
|
||
)
|
||
|
||
if server_checks:
|
||
if args.easy_mode and not args.json_stdout:
|
||
print("[4/5] 正在检查 Nginx、连接数、Docker 和服务器资源……", flush=True)
|
||
if resource_future is not None:
|
||
try:
|
||
report["server"]["host"] = resource_future.result(timeout=max(10.0, args.resource_sample_seconds + 5.0))
|
||
except Exception as exc: # diagnostic coverage must degrade, not abort the report
|
||
report["server"]["host"] = {"available": False, "error": _safe_error(exc)}
|
||
try:
|
||
docker = collect_docker_snapshot()
|
||
except Exception as exc:
|
||
docker = {"available": False, "error": _safe_error(exc)}
|
||
nginx_ids = docker.pop("nginx_container_ids", []) if isinstance(docker, dict) else []
|
||
report["server"]["docker"] = docker
|
||
relevant_ports = {80, 443, 2026, 8001, args.mysql_port}
|
||
for url in urls:
|
||
_scheme, _host, port = _origin_tuple(url)
|
||
relevant_ports.add(port)
|
||
try:
|
||
report["server"]["sockets"] = collect_socket_snapshot(relevant_ports)
|
||
except Exception as exc:
|
||
report["server"]["sockets"] = {"available": False, "error": _safe_error(exc)}
|
||
try:
|
||
report["server"]["nginx"] = collect_nginx_config(nginx_ids)
|
||
except Exception as exc:
|
||
report["server"]["nginx"] = {"available": False, "error": _safe_error(exc)}
|
||
try:
|
||
report["server"]["nginx_status"] = probe_nginx_stub_status(args.nginx_status_url, options)
|
||
except Exception as exc:
|
||
report["server"]["nginx_status"] = {"available": False, "error": _safe_error(exc)}
|
||
log_paths = discover_log_paths(args.log, config_path)
|
||
try:
|
||
report["server"]["logs"] = scan_logs(
|
||
log_paths,
|
||
tail_bytes=max(65536, min(args.log_tail_bytes, 64 * 1024 * 1024)),
|
||
)
|
||
except Exception as exc:
|
||
report["server"]["logs"] = {"available": False, "error": _safe_error(exc)}
|
||
try:
|
||
report["server"]["catalogue_scan"] = collect_catalogue_scan(config_path)
|
||
except Exception as exc:
|
||
report["server"]["catalogue_scan"] = {"available": False, "error": _safe_error(exc)}
|
||
try:
|
||
report["server"]["sqlite"] = collect_sqlite_diagnostics(config_path, config)
|
||
except Exception as exc:
|
||
report["server"]["sqlite"] = {"enabled": True, "available": False, "error": _safe_error(exc)}
|
||
mysql_explicit = bool(args.mysql_defaults_file) or bool(args.mysql_host)
|
||
mysql_enabled = args.mysql if args.mysql is not None else mysql_explicit
|
||
try:
|
||
report["server"]["mysql"] = collect_mysql_diagnostics(
|
||
enabled=mysql_enabled,
|
||
mysql_url_env=args.mysql_url_env,
|
||
defaults_file=args.mysql_defaults_file,
|
||
mysql_host=args.mysql_host,
|
||
mysql_port=args.mysql_port,
|
||
probes=max(1, min(args.mysql_probes, 10)),
|
||
)
|
||
except Exception as exc:
|
||
report["server"]["mysql"] = {"enabled": mysql_enabled, "available": False, "error": _safe_error(exc)}
|
||
finally:
|
||
if resource_executor is not None:
|
||
resource_executor.shutdown(wait=False, cancel_futures=True)
|
||
|
||
local_health = report.get("http", {}).get("local_gateway_health", {})
|
||
local_gateway_verified = bool(local_health.get("success") and local_health.get("status_code") == 200)
|
||
target_host_confirmed = bool(args.easy_target_host_confirmed)
|
||
if args.easy_mode and urls:
|
||
_target_scheme, target_host, _target_port = _origin_tuple(urls[0])
|
||
target_host_confirmed = target_host_confirmed or _host_is_loopback(target_host)
|
||
report["server"]["scope"] = {
|
||
"execution_host_only": True,
|
||
"local_gateway_verified": local_gateway_verified,
|
||
"target_host_confirmed": target_host_confirmed if args.easy_mode else None,
|
||
"confirmation": "operator confirmed execution on target host"
|
||
if target_host_confirmed
|
||
else ("not confirmed in easy mode" if args.easy_mode else "advanced mode; operator supplied scope"),
|
||
}
|
||
if args.easy_mode and not args.json_stdout:
|
||
print("[5/5] 正在汇总最可能的问题方向……", flush=True)
|
||
report["findings"] = build_findings(report)
|
||
report["quick_verdict"] = build_quick_verdict(report["findings"])
|
||
report["coverage"]["unavailable"] = _unavailable_coverage(report)
|
||
report["finished_at"] = utc_now_iso()
|
||
|
||
output_path: Path | None = planned_output_path
|
||
if output_path is not None:
|
||
write_private_json(output_path, report)
|
||
if args.json_stdout:
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
else:
|
||
print_console_summary(report, output_path)
|
||
return 0
|
||
|
||
|
||
def cli_entry() -> int:
|
||
cleanup_env_names: list[str] = []
|
||
try:
|
||
arguments, cleanup_env_names = prepare_cli_arguments(sys.argv[1:])
|
||
return main(arguments)
|
||
except KeyboardInterrupt:
|
||
print("\n诊断已取消。", file=sys.stderr)
|
||
return 130
|
||
except (EOFError, ValueError) as exc:
|
||
print(f"一键诊断无法启动: {exc}", file=sys.stderr)
|
||
return 2
|
||
finally:
|
||
for env_name in cleanup_env_names:
|
||
os.environ.pop(env_name, None)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(cli_entry())
|