代码提交
This commit is contained in:
1 parent
4ac7bc21cd
commit
17cf849361
110 files changed
+17007
-518
No files matched your search
+18
@@ -0,0 +1,18 @@
|
||||
"""Security policies for external-effect workflow nodes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
|
||||
from deerflow.workflows.security.sql_policy import (
|
||||
assert_parameters_bound,
|
||||
assert_read_only,
|
||||
enforce_limit,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"assert_parameters_bound",
|
||||
"assert_read_only",
|
||||
"enforce_http_target",
|
||||
"enforce_limit",
|
||||
"enforce_redirect",
|
||||
]
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""Egress policy for the ``http`` node (SSRF defence).
|
||||
|
||||
Checks, in order: scheme → host allow/deny lists → port → every resolved IP
|
||||
address. Resolving the host and validating *all* A/AAAA answers is what stops
|
||||
`http://internal.example.com` from pointing at `169.254.169.254` or `127.0.0.1`.
|
||||
|
||||
Known limitation: we validate the resolved addresses and then let httpx dial the
|
||||
hostname again, so a DNS answer that changes between the two lookups is not
|
||||
covered (classic DNS-rebinding TOCTOU). Closing that fully requires dialing the
|
||||
pinned IP with SNI/Host overrides; for now the deployment guidance is to keep
|
||||
``allow_private_networks`` false and prefer an explicit ``allowed_hosts`` list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse, urlsplit
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowHttpPolicyConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_ALLOWED_SCHEMES = ("http", "https")
|
||||
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
# Cloud metadata endpoints get their own explicit deny so a permissive
|
||||
# intranet configuration cannot accidentally expose instance credentials.
|
||||
_METADATA_HOSTS = frozenset({"169.254.169.254", "metadata.google.internal", "metadata"})
|
||||
|
||||
|
||||
def _denied(message: str, **details: object) -> WorkflowError:
|
||||
return WorkflowError("WORKFLOW_HTTP_POLICY_DENIED", message, details=dict(details))
|
||||
|
||||
|
||||
def _host_matches(host: str, patterns: list[str]) -> bool:
|
||||
target = host.lower().rstrip(".")
|
||||
for raw in patterns:
|
||||
pattern = str(raw).strip().lower().rstrip(".")
|
||||
if not pattern:
|
||||
continue
|
||||
if pattern.startswith("."):
|
||||
if target == pattern[1:] or target.endswith(pattern):
|
||||
return True
|
||||
elif pattern.startswith("*."):
|
||||
if target == pattern[2:] or target.endswith(pattern[1:]):
|
||||
return True
|
||||
elif target == pattern:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_private(address: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
except ValueError:
|
||||
return False
|
||||
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
|
||||
|
||||
|
||||
async def _resolve(host: str, port: int) -> list[str]:
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
infos = await loop.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"无法解析目标域名",
|
||||
retryable=True,
|
||||
details={"host": host, "reason": type(exc).__name__},
|
||||
) from exc
|
||||
return sorted({str(info[4][0]) for info in infos})
|
||||
|
||||
|
||||
async def enforce_http_target(url: str, policy: WorkflowHttpPolicyConfig) -> dict[str, object]:
|
||||
"""Validate a target URL. Returns the resolved metadata or raises."""
|
||||
if not policy.enabled:
|
||||
raise _denied("HTTP 节点已被系统禁用")
|
||||
parsed = urlparse(url)
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in _ALLOWED_SCHEMES:
|
||||
raise _denied("仅允许 http/https 协议", scheme=scheme or "")
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
raise _denied("URL 缺少主机名")
|
||||
port = parsed.port or _DEFAULT_PORTS[scheme]
|
||||
if host in _METADATA_HOSTS:
|
||||
raise _denied("目标为云元数据地址,已拒绝", host=host)
|
||||
if _host_matches(host, policy.blocked_hosts):
|
||||
raise _denied("目标主机在禁止名单中", host=host)
|
||||
if policy.allowed_hosts and not _host_matches(host, policy.allowed_hosts):
|
||||
raise _denied("目标主机不在允许名单中", host=host)
|
||||
if policy.allowed_ports and port not in policy.allowed_ports:
|
||||
raise _denied("目标端口不被允许", port=port)
|
||||
|
||||
addresses = await _resolve(host, port)
|
||||
if not policy.allow_private_networks:
|
||||
private = [addr for addr in addresses if _is_private(addr)]
|
||||
if private:
|
||||
raise _denied("目标解析到内网/环回地址,已拒绝", host=host)
|
||||
if any(addr in _METADATA_HOSTS for addr in addresses):
|
||||
raise _denied("目标解析到云元数据地址,已拒绝", host=host)
|
||||
return {"host": host, "port": port, "scheme": scheme, "addresses": addresses}
|
||||
|
||||
|
||||
async def enforce_redirect(location: str, base_url: str, policy: WorkflowHttpPolicyConfig) -> str:
|
||||
"""Validate a redirect hop, resolving relative locations against the base."""
|
||||
if location.startswith("/"):
|
||||
base = urlsplit(base_url)
|
||||
location = f"{base.scheme}://{base.netloc}{location}"
|
||||
await enforce_http_target(location, policy)
|
||||
return location
|
||||
|
||||
|
||||
__all__ = ["enforce_http_target", "enforce_redirect"]
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
"""Envelope encryption for workflow secrets (data-source DSNs, credentials).
|
||||
|
||||
The key comes from ``WORKFLOW_SECRET_KEY`` (or ``DEERFLOW_SECRET_KEY``) and is
|
||||
stretched with SHA-256 into a Fernet key, so operators can supply any
|
||||
sufficiently long passphrase.
|
||||
|
||||
When no key is configured we refuse to encrypt rather than silently storing
|
||||
plaintext: the caller decides whether to reject the write or fall back. That
|
||||
choice is deliberate — a silent plaintext fallback is how credentials end up in
|
||||
database backups without anyone noticing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENV_KEYS = ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY")
|
||||
_PREFIX = "enc:v1:"
|
||||
|
||||
|
||||
class SecretKeyMissingError(RuntimeError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("未配置 WORKFLOW_SECRET_KEY,无法加密保存敏感连接信息")
|
||||
|
||||
|
||||
def _fernet():
|
||||
key_material = ""
|
||||
for name in _ENV_KEYS:
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
key_material = value.strip()
|
||||
break
|
||||
if not key_material:
|
||||
raise SecretKeyMissingError()
|
||||
try:
|
||||
from cryptography.fernet import Fernet
|
||||
except ImportError as exc: # pragma: no cover - dependency is declared
|
||||
raise SecretKeyMissingError() from exc
|
||||
digest = hashlib.sha256(key_material.encode("utf-8")).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
|
||||
def secrets_available() -> bool:
|
||||
try:
|
||||
_fernet()
|
||||
except Exception: # noqa: BLE001 - any failure means "cannot encrypt"
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def encrypt_secret(plaintext: str) -> str:
|
||||
token = _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
|
||||
return f"{_PREFIX}{token}"
|
||||
|
||||
|
||||
def decrypt_secret(stored: str) -> str:
|
||||
if not stored:
|
||||
return ""
|
||||
if not stored.startswith(_PREFIX):
|
||||
# Legacy / externally seeded plaintext row.
|
||||
return stored
|
||||
return _fernet().decrypt(stored[len(_PREFIX) :].encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
def is_encrypted(stored: str) -> bool:
|
||||
return bool(stored) and stored.startswith(_PREFIX)
|
||||
|
||||
|
||||
def mask_dsn(dsn: str) -> str:
|
||||
"""Human-readable summary that never reveals user/password/host details."""
|
||||
if not dsn:
|
||||
return ""
|
||||
scheme, _, rest = dsn.partition("://")
|
||||
if not rest:
|
||||
return f"{scheme[:24]}://***"
|
||||
host_part = rest.split("@")[-1]
|
||||
host = host_part.split("/")[0]
|
||||
database = host_part.split("/", 1)[1].split("?")[0] if "/" in host_part else ""
|
||||
masked_host = host.split(":")[0]
|
||||
if len(masked_host) > 4:
|
||||
masked_host = f"{masked_host[:2]}***{masked_host[-2:]}"
|
||||
return f"{scheme}://***@{masked_host}/{database}"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SecretKeyMissingError",
|
||||
"decrypt_secret",
|
||||
"encrypt_secret",
|
||||
"is_encrypted",
|
||||
"mask_dsn",
|
||||
"secrets_available",
|
||||
]
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
"""Read-only statement policy for the ``sql_read`` node.
|
||||
|
||||
The node is *only* allowed to run a single read query. We reject anything that
|
||||
is not a `SELECT`/`WITH`, anything containing a statement separator, and any
|
||||
write/DDL/administrative keyword — after stripping comments, because
|
||||
``SELECT 1 -- ; DROP TABLE t`` and ``/*!*/`` style tricks are exactly how these
|
||||
filters get bypassed.
|
||||
|
||||
Parameters are always bound (never interpolated), so this policy is a second
|
||||
line of defence rather than the only one. Production deployments should also
|
||||
point the data source at a read-only database account.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_FORBIDDEN_KEYWORDS = (
|
||||
"insert",
|
||||
"update",
|
||||
"delete",
|
||||
"drop",
|
||||
"alter",
|
||||
"create",
|
||||
"truncate",
|
||||
"replace into",
|
||||
"grant",
|
||||
"revoke",
|
||||
"merge",
|
||||
"call",
|
||||
"execute",
|
||||
"exec",
|
||||
"copy",
|
||||
"load_file",
|
||||
"outfile",
|
||||
"dumpfile",
|
||||
"into outfile",
|
||||
"pg_read_file",
|
||||
"pg_sleep",
|
||||
"sleep",
|
||||
"benchmark",
|
||||
"shutdown",
|
||||
"vacuum",
|
||||
"attach",
|
||||
"detach",
|
||||
"pragma",
|
||||
"set ",
|
||||
"lock",
|
||||
"do ",
|
||||
)
|
||||
|
||||
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
|
||||
_LINE_COMMENT = re.compile(r"(--|#)[^\n]*")
|
||||
_PARAM_RE = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)")
|
||||
|
||||
|
||||
def _denied(message: str, **details: object) -> WorkflowError:
|
||||
return WorkflowError("WORKFLOW_SQL_POLICY_DENIED", message, details=dict(details))
|
||||
|
||||
|
||||
def strip_comments(statement: str) -> str:
|
||||
return _LINE_COMMENT.sub(" ", _BLOCK_COMMENT.sub(" ", statement))
|
||||
|
||||
|
||||
def assert_read_only(statement: str) -> str:
|
||||
"""Validate and return the normalised single-statement query."""
|
||||
raw = (statement or "").strip()
|
||||
if not raw:
|
||||
raise _denied("SQL 语句为空")
|
||||
cleaned = strip_comments(raw).strip()
|
||||
if not cleaned:
|
||||
raise _denied("SQL 语句仅包含注释")
|
||||
body = cleaned.rstrip().rstrip(";").strip()
|
||||
if ";" in body:
|
||||
raise _denied("只允许执行单条语句")
|
||||
normalized = re.sub(r"\s+", " ", body.lower()).strip()
|
||||
if not (normalized.startswith("select") or normalized.startswith("with")):
|
||||
raise _denied("只允许 SELECT / WITH 查询")
|
||||
for keyword in _FORBIDDEN_KEYWORDS:
|
||||
token = keyword.strip()
|
||||
if re.search(rf"\b{re.escape(token)}\b", normalized):
|
||||
raise _denied("语句包含被禁止的关键字", keyword=token)
|
||||
return body
|
||||
|
||||
|
||||
def enforce_limit(statement: str, max_rows: int) -> str:
|
||||
"""Append a LIMIT when the query does not already bound its result size."""
|
||||
lowered = statement.lower()
|
||||
if re.search(r"\blimit\b", lowered) or re.search(r"\bfetch\s+first\b", lowered):
|
||||
return statement
|
||||
return f"{statement} LIMIT {int(max_rows)}"
|
||||
|
||||
|
||||
_FROM_JOIN_RE = re.compile(r"\b(?:from|join)\s+([A-Za-z_][\w.]*)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _table_name(raw: str) -> str:
|
||||
return raw.split(".")[-1].strip("`\"[]").lower()
|
||||
|
||||
|
||||
def assert_tables_allowed(statement: str, allowed_tables: list[str]) -> None:
|
||||
"""When the data source has a table allowlist, every FROM/JOIN target must
|
||||
appear on it. An empty allowlist means no extra restriction."""
|
||||
if not allowed_tables:
|
||||
return
|
||||
allowed = {item.lower() for item in allowed_tables} | {_table_name(item) for item in allowed_tables}
|
||||
for name in _FROM_JOIN_RE.findall(strip_comments(statement)):
|
||||
short = _table_name(name)
|
||||
if short not in allowed and name.lower() not in allowed:
|
||||
raise _denied("查询引用了未授权的表", table=name, allowedTables=list(allowed_tables))
|
||||
|
||||
|
||||
def declared_parameters(statement: str) -> set[str]:
|
||||
return set(_PARAM_RE.findall(strip_comments(statement)))
|
||||
|
||||
|
||||
def assert_parameters_bound(statement: str, parameters: dict[str, object]) -> None:
|
||||
missing = sorted(declared_parameters(statement) - set(parameters))
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
f"缺少查询参数:{'、'.join(missing)}",
|
||||
details={"missing": missing},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"assert_parameters_bound",
|
||||
"assert_read_only",
|
||||
"assert_tables_allowed",
|
||||
"declared_parameters",
|
||||
"enforce_limit",
|
||||
"strip_comments",
|
||||
]
|
||||
Reference in new issue
Block a user