初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,18 @@
"""Security policies for external-effect workflow nodes."""
from __future__ import annotations
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
enforce_limit,
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"enforce_http_target",
"enforce_limit",
"enforce_redirect",
]
@@ -0,0 +1,114 @@
"""Egress policy for the ``http`` node (SSRF defence).
Checks, in order: scheme → host allow/deny lists → port → every resolved IP
address. Resolving the host and validating *all* A/AAAA answers is what stops
`http://internal.example.com` from pointing at `169.254.169.254` or `127.0.0.1`.
Known limitation: we validate the resolved addresses and then let httpx dial the
hostname again, so a DNS answer that changes between the two lookups is not
covered (classic DNS-rebinding TOCTOU). Closing that fully requires dialing the
pinned IP with SNI/Host overrides; for now the deployment guidance is to keep
``allow_private_networks`` false and prefer an explicit ``allowed_hosts`` list.
"""
from __future__ import annotations
import asyncio
import ipaddress
import socket
from urllib.parse import urlparse, urlsplit
from deerflow.config.workflow_config import WorkflowHttpPolicyConfig
from deerflow.workflows.errors import WorkflowError
_ALLOWED_SCHEMES = ("http", "https")
_DEFAULT_PORTS = {"http": 80, "https": 443}
# Cloud metadata endpoints get their own explicit deny so a permissive
# intranet configuration cannot accidentally expose instance credentials.
_METADATA_HOSTS = frozenset({"169.254.169.254", "metadata.google.internal", "metadata"})
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_HTTP_POLICY_DENIED", message, details=dict(details))
def _host_matches(host: str, patterns: list[str]) -> bool:
target = host.lower().rstrip(".")
for raw in patterns:
pattern = str(raw).strip().lower().rstrip(".")
if not pattern:
continue
if pattern.startswith("."):
if target == pattern[1:] or target.endswith(pattern):
return True
elif pattern.startswith("*."):
if target == pattern[2:] or target.endswith(pattern[1:]):
return True
elif target == pattern:
return True
return False
def _is_private(address: str) -> bool:
try:
ip = ipaddress.ip_address(address)
except ValueError:
return False
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
async def _resolve(host: str, port: int) -> list[str]:
loop = asyncio.get_running_loop()
try:
infos = await loop.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"无法解析目标域名",
retryable=True,
details={"host": host, "reason": type(exc).__name__},
) from exc
return sorted({str(info[4][0]) for info in infos})
async def enforce_http_target(url: str, policy: WorkflowHttpPolicyConfig) -> dict[str, object]:
"""Validate a target URL. Returns the resolved metadata or raises."""
if not policy.enabled:
raise _denied("HTTP 节点已被系统禁用")
parsed = urlparse(url)
scheme = (parsed.scheme or "").lower()
if scheme not in _ALLOWED_SCHEMES:
raise _denied("仅允许 http/https 协议", scheme=scheme or "")
host = (parsed.hostname or "").lower()
if not host:
raise _denied("URL 缺少主机名")
port = parsed.port or _DEFAULT_PORTS[scheme]
if host in _METADATA_HOSTS:
raise _denied("目标为云元数据地址,已拒绝", host=host)
if _host_matches(host, policy.blocked_hosts):
raise _denied("目标主机在禁止名单中", host=host)
if policy.allowed_hosts and not _host_matches(host, policy.allowed_hosts):
raise _denied("目标主机不在允许名单中", host=host)
if policy.allowed_ports and port not in policy.allowed_ports:
raise _denied("目标端口不被允许", port=port)
addresses = await _resolve(host, port)
if not policy.allow_private_networks:
private = [addr for addr in addresses if _is_private(addr)]
if private:
raise _denied("目标解析到内网/环回地址,已拒绝", host=host)
if any(addr in _METADATA_HOSTS for addr in addresses):
raise _denied("目标解析到云元数据地址,已拒绝", host=host)
return {"host": host, "port": port, "scheme": scheme, "addresses": addresses}
async def enforce_redirect(location: str, base_url: str, policy: WorkflowHttpPolicyConfig) -> str:
"""Validate a redirect hop, resolving relative locations against the base."""
if location.startswith("/"):
base = urlsplit(base_url)
location = f"{base.scheme}://{base.netloc}{location}"
await enforce_http_target(location, policy)
return location
__all__ = ["enforce_http_target", "enforce_redirect"]
@@ -0,0 +1,97 @@
"""Envelope encryption for workflow secrets (data-source DSNs, credentials).
The key comes from ``WORKFLOW_SECRET_KEY`` (or ``DEERFLOW_SECRET_KEY``) and is
stretched with SHA-256 into a Fernet key, so operators can supply any
sufficiently long passphrase.
When no key is configured we refuse to encrypt rather than silently storing
plaintext: the caller decides whether to reject the write or fall back. That
choice is deliberate — a silent plaintext fallback is how credentials end up in
database backups without anyone noticing.
"""
from __future__ import annotations
import base64
import hashlib
import logging
import os
logger = logging.getLogger(__name__)
_ENV_KEYS = ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY")
_PREFIX = "enc:v1:"
class SecretKeyMissingError(RuntimeError):
def __init__(self) -> None:
super().__init__("未配置 WORKFLOW_SECRET_KEY,无法加密保存敏感连接信息")
def _fernet():
key_material = ""
for name in _ENV_KEYS:
value = os.environ.get(name)
if value and value.strip():
key_material = value.strip()
break
if not key_material:
raise SecretKeyMissingError()
try:
from cryptography.fernet import Fernet
except ImportError as exc: # pragma: no cover - dependency is declared
raise SecretKeyMissingError() from exc
digest = hashlib.sha256(key_material.encode("utf-8")).digest()
return Fernet(base64.urlsafe_b64encode(digest))
def secrets_available() -> bool:
try:
_fernet()
except Exception: # noqa: BLE001 - any failure means "cannot encrypt"
return False
return True
def encrypt_secret(plaintext: str) -> str:
token = _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
return f"{_PREFIX}{token}"
def decrypt_secret(stored: str) -> str:
if not stored:
return ""
if not stored.startswith(_PREFIX):
# Legacy / externally seeded plaintext row.
return stored
return _fernet().decrypt(stored[len(_PREFIX) :].encode("ascii")).decode("utf-8")
def is_encrypted(stored: str) -> bool:
return bool(stored) and stored.startswith(_PREFIX)
def mask_dsn(dsn: str) -> str:
"""Human-readable summary that never reveals user/password/host details."""
if not dsn:
return ""
scheme, _, rest = dsn.partition("://")
if not rest:
return f"{scheme[:24]}://***"
host_part = rest.split("@")[-1]
host = host_part.split("/")[0]
database = host_part.split("/", 1)[1].split("?")[0] if "/" in host_part else ""
masked_host = host.split(":")[0]
if len(masked_host) > 4:
masked_host = f"{masked_host[:2]}***{masked_host[-2:]}"
return f"{scheme}://***@{masked_host}/{database}"
__all__ = [
"SecretKeyMissingError",
"decrypt_secret",
"encrypt_secret",
"is_encrypted",
"mask_dsn",
"secrets_available",
]
@@ -0,0 +1,136 @@
"""Read-only statement policy for the ``sql_read`` node.
The node is *only* allowed to run a single read query. We reject anything that
is not a `SELECT`/`WITH`, anything containing a statement separator, and any
write/DDL/administrative keyword — after stripping comments, because
``SELECT 1 -- ; DROP TABLE t`` and ``/*!*/`` style tricks are exactly how these
filters get bypassed.
Parameters are always bound (never interpolated), so this policy is a second
line of defence rather than the only one. Production deployments should also
point the data source at a read-only database account.
"""
from __future__ import annotations
import re
from deerflow.workflows.errors import WorkflowError
_FORBIDDEN_KEYWORDS = (
"insert",
"update",
"delete",
"drop",
"alter",
"create",
"truncate",
"replace into",
"grant",
"revoke",
"merge",
"call",
"execute",
"exec",
"copy",
"load_file",
"outfile",
"dumpfile",
"into outfile",
"pg_read_file",
"pg_sleep",
"sleep",
"benchmark",
"shutdown",
"vacuum",
"attach",
"detach",
"pragma",
"set ",
"lock",
"do ",
)
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
_LINE_COMMENT = re.compile(r"(--|#)[^\n]*")
_PARAM_RE = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)")
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_SQL_POLICY_DENIED", message, details=dict(details))
def strip_comments(statement: str) -> str:
return _LINE_COMMENT.sub(" ", _BLOCK_COMMENT.sub(" ", statement))
def assert_read_only(statement: str) -> str:
"""Validate and return the normalised single-statement query."""
raw = (statement or "").strip()
if not raw:
raise _denied("SQL 语句为空")
cleaned = strip_comments(raw).strip()
if not cleaned:
raise _denied("SQL 语句仅包含注释")
body = cleaned.rstrip().rstrip(";").strip()
if ";" in body:
raise _denied("只允许执行单条语句")
normalized = re.sub(r"\s+", " ", body.lower()).strip()
if not (normalized.startswith("select") or normalized.startswith("with")):
raise _denied("只允许 SELECT / WITH 查询")
for keyword in _FORBIDDEN_KEYWORDS:
token = keyword.strip()
if re.search(rf"\b{re.escape(token)}\b", normalized):
raise _denied("语句包含被禁止的关键字", keyword=token)
return body
def enforce_limit(statement: str, max_rows: int) -> str:
"""Append a LIMIT when the query does not already bound its result size."""
lowered = statement.lower()
if re.search(r"\blimit\b", lowered) or re.search(r"\bfetch\s+first\b", lowered):
return statement
return f"{statement} LIMIT {int(max_rows)}"
_FROM_JOIN_RE = re.compile(r"\b(?:from|join)\s+([A-Za-z_][\w.]*)", re.IGNORECASE)
def _table_name(raw: str) -> str:
return raw.split(".")[-1].strip("`\"[]").lower()
def assert_tables_allowed(statement: str, allowed_tables: list[str]) -> None:
"""When the data source has a table allowlist, every FROM/JOIN target must
appear on it. An empty allowlist means no extra restriction."""
if not allowed_tables:
return
allowed = {item.lower() for item in allowed_tables} | {_table_name(item) for item in allowed_tables}
for name in _FROM_JOIN_RE.findall(strip_comments(statement)):
short = _table_name(name)
if short not in allowed and name.lower() not in allowed:
raise _denied("查询引用了未授权的表", table=name, allowedTables=list(allowed_tables))
def declared_parameters(statement: str) -> set[str]:
return set(_PARAM_RE.findall(strip_comments(statement)))
def assert_parameters_bound(statement: str, parameters: dict[str, object]) -> None:
missing = sorted(declared_parameters(statement) - set(parameters))
if missing:
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
f"缺少查询参数:{'、'.join(missing)}",
details={"missing": missing},
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"assert_tables_allowed",
"declared_parameters",
"enforce_limit",
"strip_comments",
]