Files

711 lines
30 KiB
Python

import errno
import logging
import ntpath
import os
import shutil
import signal
import stat
import subprocess
import tempfile
import time
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import BinaryIO, NamedTuple
from deerflow.sandbox.local.list_dir import list_dir
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class PathMapping:
"""A path mapping from a container path to a local path with optional read-only flag."""
container_path: str
local_path: str
read_only: bool = False
class ResolvedPath(NamedTuple):
path: str
mapping: PathMapping | None
class LocalSandbox(Sandbox):
_COMMAND_TIMEOUT_SECONDS = 600
_COMMAND_CLEANUP_SECONDS = 5
# Tool-level truncation happens after execution. Never first materialize an
# arbitrarily large command response in the Gateway process.
_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024
_COMMAND_CAPTURE_MAX_BYTES = 16 * 1024 * 1024
_MODEL_COMMAND_HIDDEN_ENV_KEYS = frozenset(
{
"DEER_FLOW_CONFIG_PATH",
"DEER_FLOW_EXTENSIONS_CONFIG_PATH",
"DEER_FLOW_HOME",
"DEER_FLOW_HOST_BASE_DIR",
"DEER_FLOW_HOST_SKILLS_PATH",
"DEER_FLOW_PROJECT_ROOT",
"DEER_FLOW_SKILLS_PATH",
"PYTHONPATH",
}
)
@staticmethod
def _model_command_environment() -> dict[str, str]:
"""Return a reduced environment for model-originated host commands.
LocalSandbox is deliberately not an OS security sandbox, but it should
not hand a model canonical skills/config paths through process
environment variables. ``PYTHONPATH`` is also removed so an editable
backend checkout is not advertised as an import root. Non-path
``DEER_FLOW_*`` settings (for example the Gateway port), ordinary PATH,
proxy and skill-specific credentials remain available.
"""
return {key: value for key, value in os.environ.items() if key.upper() not in LocalSandbox._MODEL_COMMAND_HIDDEN_ENV_KEYS}
@staticmethod
def _shell_name(shell: str) -> str:
"""Return the executable name for a shell path or command."""
return shell.replace("\\", "/").rsplit("/", 1)[-1].lower()
@staticmethod
def _is_powershell(shell: str) -> bool:
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
return shell
continue
shell_from_path = shutil.which(shell)
if shell_from_path is not None:
return shell_from_path
return None
def __init__(self, id: str, path_mappings: list[PathMapping] | None = None):
"""
Initialize local sandbox with optional path mappings.
Args:
id: Sandbox identifier
path_mappings: List of path mappings with optional read-only flag.
Skills directory is read-only by default.
"""
super().__init__(id)
self.path_mappings = path_mappings or []
# Track files written through write_file so read_file only
# reverse-resolves paths in agent-authored content.
self._agent_written_paths: set[str] = set()
def _is_read_only_path(self, resolved_path: str) -> bool:
"""Check if a resolved path is under a read-only mount.
When multiple mappings match (nested mounts), prefer the most specific
mapping (i.e. the one whose local_path is the longest prefix of the
resolved path), similar to how ``_resolve_path`` handles container paths.
"""
resolved = str(Path(resolved_path).resolve())
best_mapping: PathMapping | None = None
best_prefix_len = -1
for mapping in self.path_mappings:
local_resolved = str(Path(mapping.local_path).resolve())
if resolved == local_resolved or resolved.startswith(local_resolved + os.sep):
prefix_len = len(local_resolved)
if prefix_len > best_prefix_len:
best_prefix_len = prefix_len
best_mapping = mapping
if best_mapping is None:
return False
return best_mapping.read_only
def _find_path_mapping(self, path: str) -> tuple[PathMapping, str] | None:
path_str = str(path)
for mapping in sorted(self.path_mappings, key=lambda m: len(m.container_path.rstrip("/") or "/"), reverse=True):
container_path = mapping.container_path.rstrip("/") or "/"
if container_path == "/":
if path_str.startswith("/"):
return mapping, path_str.lstrip("/")
continue
if path_str == container_path or path_str.startswith(container_path + "/"):
relative = path_str[len(container_path) :].lstrip("/")
return mapping, relative
return None
def _resolve_path_with_mapping(self, path: str) -> ResolvedPath:
"""
Resolve container path to actual local path using mappings.
Args:
path: Path that might be a container path
Returns:
Resolved local path and the matched mapping, if any
"""
path_str = str(path)
mapping_match = self._find_path_mapping(path_str)
if mapping_match is None:
return ResolvedPath(path_str, None)
mapping, relative = mapping_match
local_root = Path(mapping.local_path).resolve()
resolved_path = (local_root / relative).resolve() if relative else local_root
try:
resolved_path.relative_to(local_root)
except ValueError as exc:
raise PermissionError(errno.EACCES, "Access denied: path escapes mounted directory", path_str) from exc
return ResolvedPath(str(resolved_path), mapping)
def _resolve_path(self, path: str) -> str:
return self._resolve_path_with_mapping(path).path
def _is_resolved_path_read_only(self, resolved: ResolvedPath) -> bool:
return bool(resolved.mapping and resolved.mapping.read_only) or self._is_read_only_path(resolved.path)
def _reverse_resolve_path(self, path: str) -> str:
"""
Reverse resolve local path back to container path using mappings.
Args:
path: Local path that might need to be mapped to container path
Returns:
Container path if mapping exists, otherwise original path
"""
normalized_path = path.replace("\\", "/")
path_str = str(Path(normalized_path).resolve())
# Try each mapping (longest local path first for more specific matches)
for mapping in sorted(self.path_mappings, key=lambda m: len(m.local_path), reverse=True):
local_path_resolved = str(Path(mapping.local_path).resolve())
if path_str == local_path_resolved or path_str.startswith(local_path_resolved + "/"):
# Replace the local path prefix with container path
relative = path_str[len(local_path_resolved) :].lstrip("/")
resolved = f"{mapping.container_path}/{relative}" if relative else mapping.container_path
return resolved
# No mapping found, return original path
return path_str
def _reverse_resolve_paths_in_output(self, output: str) -> str:
"""
Reverse resolve local paths back to container paths in output string.
Args:
output: Output string that may contain local paths
Returns:
Output with local paths resolved to container paths
"""
import re
# Sort mappings by local path length (longest first) for correct prefix matching
sorted_mappings = sorted(self.path_mappings, key=lambda m: len(m.local_path), reverse=True)
if not sorted_mappings:
return output
# Create pattern that matches absolute paths
# Match paths like /Users/... or other absolute paths
result = output
for mapping in sorted_mappings:
# Escape the local path for use in regex
escaped_local = re.escape(str(Path(mapping.local_path).resolve()))
# Match the local path followed by optional path components with either separator
pattern = re.compile(escaped_local + r"(?:[/\\][^\s\"';&|<>()]*)?")
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
return self._reverse_resolve_path(matched_path)
result = pattern.sub(replace_match, result)
return result
def _resolve_paths_in_command(self, command: str) -> str:
"""
Resolve container paths to local paths in a command string.
Args:
command: Command string that may contain container paths
Returns:
Command with container paths resolved to local paths
"""
import re
# Sort mappings by length (longest first) for correct prefix matching
sorted_mappings = sorted(self.path_mappings, key=lambda m: len(m.container_path), reverse=True)
# Build regex pattern to match all container paths
# Match container path followed by optional path components
if not sorted_mappings:
return command
# Create pattern that matches any of the container paths.
# The lookahead (?=/|$|...) ensures we only match at a path-segment boundary,
# preventing /mnt/skills from matching inside /mnt/skills-extra.
patterns = [re.escape(m.container_path) + r"(?=/|$|[\s\"';&|<>()])(?:/[^\s\"';&|<>()]*)?" for m in sorted_mappings]
pattern = re.compile("|".join(f"({p})" for p in patterns))
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
resolved = self._resolve_path(matched_path)
# Normalize to forward slashes: on Windows hosts the resolved path
# contains backslashes that the shell (bash -c) would treat as
# escape characters, corrupting the rewritten command (e.g.
# F:\repo\... losing its separators). Same normalization as
# ``_resolve_paths_in_content``; bash and Windows tools all accept
# forward-slash paths.
return resolved.replace("\\", "/")
return pattern.sub(replace_match, command)
def _resolve_paths_in_content(self, content: str) -> str:
"""Resolve container paths to local paths in arbitrary file content.
Unlike ``_resolve_paths_in_command`` which uses shell-aware boundary
characters, this method treats the content as plain text and resolves
every occurrence of a container path prefix. Resolved paths are
normalized to forward slashes to avoid backslash-escape issues on
Windows hosts (e.g. ``C:\\Users\\..`` breaking Python string literals).
Args:
content: File content that may contain container paths.
Returns:
Content with container paths resolved to local paths (forward slashes).
"""
import re
sorted_mappings = sorted(self.path_mappings, key=lambda m: len(m.container_path), reverse=True)
if not sorted_mappings:
return content
patterns = [re.escape(m.container_path) + r"(?=/|$|[^\w./-])(?:/[^\s\"';&|<>()]*)?" for m in sorted_mappings]
pattern = re.compile("|".join(f"({p})" for p in patterns))
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
resolved = self._resolve_path(matched_path)
# Normalize to forward slashes so that Windows backslash paths
# don't create invalid escape sequences in source files.
return resolved.replace("\\", "/")
return pattern.sub(replace_match, content)
@staticmethod
def _get_shell() -> str:
"""Detect available shell executable with fallback."""
shell = LocalSandbox._find_first_available_shell(("/bin/zsh", "/bin/bash", "/bin/sh", "sh"))
if shell is not None:
return shell
if os.name == "nt":
system_root = os.environ.get("SystemRoot", r"C:\Windows")
shell = LocalSandbox._find_first_available_shell(
(
"pwsh",
"pwsh.exe",
"powershell",
"powershell.exe",
ntpath.join(system_root, "System32", "WindowsPowerShell", "v1.0", "powershell.exe"),
"cmd.exe",
)
)
if shell is not None:
return shell
raise RuntimeError("No suitable shell executable found. Tried /bin/zsh, /bin/bash, /bin/sh, `sh` on PATH, then PowerShell and cmd.exe fallbacks for Windows.")
raise RuntimeError("No suitable shell executable found. Tried /bin/zsh, /bin/bash, /bin/sh, and `sh` on PATH.")
def execute_command(self, command: str) -> str:
# Resolve container paths in command before execution
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-NonInteractive",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
args = [shell, "-c", resolved_command]
else:
args = [shell, "-c", resolved_command]
# Do not use PIPE + subprocess.run(timeout=...). Killing only the shell
# leaves skill subprocesses alive; on Windows run() then calls an
# unbounded communicate() while those children still hold the pipes.
# File-backed output also makes shell exit independent of a background
# child's inherited stdout/stderr and bounds Python memory usage.
timeout = self._command_timeout_seconds()
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
process = subprocess.Popen(
args,
shell=False,
stdin=subprocess.DEVNULL,
stdout=stdout,
stderr=stderr,
start_new_session=os.name != "nt",
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
env=self._model_command_environment(),
)
failure: str | None = None
try:
deadline = time.monotonic() + timeout
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(args, timeout)
try:
process.wait(timeout=min(0.1, remaining))
break
except subprocess.TimeoutExpired:
# Avoid turning an unbounded in-memory PIPE into an
# unbounded temporary file (for example `yes`).
if os.fstat(stdout.fileno()).st_size + os.fstat(stderr.fileno()).st_size > self._COMMAND_CAPTURE_MAX_BYTES:
failure = f"Command output exceeded {self._COMMAND_CAPTURE_MAX_BYTES} bytes"
self._terminate_command_tree(process)
break
except subprocess.TimeoutExpired:
failure = f"Command timed out after {timeout:g} seconds"
self._terminate_command_tree(process)
except BaseException:
self._terminate_command_tree(process)
raise
output = self._read_command_output(stdout)
error_output = self._read_command_output(stderr)
if error_output:
output += f"\nStd Error:\n{error_output}" if output else error_output
if failure:
output = f"Error: {failure}; process tree termination requested.\n{output}"
elif process.returncode != 0:
output += f"\nExit Code: {process.returncode}"
final_output = output if output else "(no output)"
# Reverse resolve local paths back to container paths in output
return self._reverse_resolve_paths_in_output(final_output)
def _command_timeout_seconds(self) -> float:
try:
from deerflow.config import get_app_config
return get_app_config().sandbox.command_timeout_seconds
except Exception:
return self._COMMAND_TIMEOUT_SECONDS
@classmethod
def _read_command_output(cls, handle: BinaryIO) -> str:
size = handle.seek(0, os.SEEK_END)
limit = cls._COMMAND_OUTPUT_MAX_BYTES
handle.seek(0)
if size <= limit:
return handle.read(limit).decode("utf-8", errors="replace")
head = handle.read(limit // 2).decode("utf-8", errors="replace")
handle.seek(-limit // 2, os.SEEK_END)
tail = handle.read(limit // 2).decode("utf-8", errors="replace")
return f"{head}\n... [command output truncated; {size} bytes total] ...\n{tail}"
@classmethod
def _terminate_command_tree(cls, process: subprocess.Popen) -> None:
"""Kill descendants before reaping the shell, with bounded cleanup."""
try:
if os.name == "nt":
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=cls._COMMAND_CLEANUP_SECONDS,
check=False,
creationflags=subprocess.CREATE_NO_WINDOW,
)
else:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except (OSError, subprocess.TimeoutExpired):
logger.warning("Failed to terminate sandbox command process tree (pid=%s)", process.pid, exc_info=True)
finally:
if process.poll() is None:
try:
process.kill()
except ProcessLookupError:
pass
try:
process.wait(timeout=cls._COMMAND_CLEANUP_SECONDS)
except subprocess.TimeoutExpired:
logger.warning("Sandbox command did not exit after termination (pid=%s)", process.pid)
def list_dir(self, path: str, max_depth=2) -> list[str]:
resolved_path = self._resolve_path(path)
entries = list_dir(resolved_path, max_depth)
# Reverse resolve local paths back to container paths and preserve
# list_dir's trailing "/" marker for directories.
result: list[str] = []
for entry in entries:
is_dir = entry.endswith(("/", "\\"))
reversed_entry = self._reverse_resolve_path(entry.rstrip("/\\")) if is_dir else self._reverse_resolve_path(entry)
result.append(f"{reversed_entry}/" if is_dir and not reversed_entry.endswith("/") else reversed_entry)
return result
def read_file(self, path: str) -> str:
resolved_path = self._resolve_path(path)
try:
with self._open_regular_text(resolved_path) as f:
content = f.read()
# Only reverse-resolve paths in files that were previously written
# by write_file (agent-authored content). User-uploaded files,
# external tool output, and other non-agent content should not be
# silently rewritten — see discussion on PR #1935.
if resolved_path in self._agent_written_paths:
content = self._reverse_resolve_paths_in_output(content)
return content
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
@contextmanager
def _open_regular_text(path: str, *, errors: str = "strict", newline: str | None = None):
# A skill script can create a FIFO inside its workspace. Opening it
# before checking the descriptor type would wait forever for a writer.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0))
try:
mode = os.fstat(fd).st_mode
if stat.S_ISDIR(mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory, not a file", path)
if not stat.S_ISREG(mode):
raise OSError(errno.EINVAL, "Only regular files can be read", path)
with os.fdopen(fd, "r", encoding="utf-8", errors=errors, newline=newline, closefd=False) as handle:
yield handle
finally:
os.close(fd)
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Stream a bounded range without materializing the whole file."""
resolved_path = self._resolve_path(path)
lines: list[str] = []
chars = 0
next_line: int | None = None
start_line = max(1, start_line)
try:
with self._open_regular_text(resolved_path, errors="replace", newline="") as handle:
number = 0
while True:
fragment = handle.readline(4097)
if not fragment:
break
number += 1
remaining = max_chars - chars - (1 if lines else 0)
capture = number >= start_line and (end_line is None or number <= end_line)
if end_line is not None and number > end_line:
break
if capture and remaining <= 0:
next_line = number
break
parts: list[str] = []
captured = 0
over_budget = False
while True:
ended = fragment.endswith(("\n", "\r"))
body = fragment.rstrip("\r\n") if ended else fragment
if capture and remaining > captured:
piece = body[: remaining - captured]
parts.append(piece)
captured += len(piece)
if len(piece) < len(body):
over_budget = True
elif capture and body:
over_budget = True
# The next request starts from the file's beginning,
# so continuation does not require draining the rest
# of an arbitrarily large physical line now.
if ended or (capture and captured >= remaining):
break
fragment = handle.readline(4097)
if not fragment:
break
if not capture:
continue
line = "".join(parts)
lines.append(line)
chars += len(line) + (1 if len(lines) > 1 else 0)
if over_budget or chars >= max_chars:
next_line = number + 1
break
content = "\n".join(lines)
if resolved_path in self._agent_written_paths:
content = self._reverse_resolve_paths_in_output(content)
return content, next_line, next_line is not None
except OSError as e:
raise type(e)(e.errno, e.strerror, path) from None
def write_file(self, path: str, content: str, append: bool = False) -> None:
resolved = self._resolve_path_with_mapping(path)
resolved_path = resolved.path
if self._is_resolved_path_read_only(resolved):
raise OSError(errno.EROFS, "Read-only file system", path)
try:
dir_path = os.path.dirname(resolved_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
# Resolve container paths in content to local paths
# using the content-specific resolver (forward-slash safe)
resolved_content = self._resolve_paths_in_content(content)
if append:
with open(resolved_path, "a", encoding="utf-8") as f:
f.write(resolved_content)
else:
self._atomic_overwrite(resolved_path, resolved_content)
# Track this path so read_file knows to reverse-resolve on read.
# Only agent-written files get reverse-resolved; user uploads and
# external tool output are left untouched.
self._agent_written_paths.add(resolved_path)
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
def _atomic_overwrite(path: str, content: str) -> None:
"""Publish a complete file only after encoding and disk writes succeed."""
# Preserve ordinary open(..., "w") symlink-following semantics.
target = os.path.realpath(path)
previous_mode: int | None = None
try:
previous_stat = os.stat(target)
except FileNotFoundError:
pass
else:
if stat.S_ISDIR(previous_stat.st_mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory", path)
if not stat.S_ISREG(previous_stat.st_mode):
raise OSError(errno.EINVAL, "Target is not a regular file", path)
# Replacement depends on directory permissions; also enforce the
# existing file's write permissions, without truncating it.
descriptor = os.open(target, os.O_WRONLY)
os.close(descriptor)
previous_mode = stat.S_IMODE(previous_stat.st_mode)
staged_path: str | None = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=os.path.dirname(target),
prefix=".deerflow-write-", suffix=".tmp", delete=False,
) as staged:
staged_path = staged.name
staged.write(content)
staged.flush()
os.fsync(staged.fileno())
if previous_mode is not None:
os.chmod(staged_path, previous_mode)
os.replace(staged_path, target)
staged_path = None
finally:
if staged_path is not None:
try:
os.unlink(staged_path)
except OSError:
logger.warning("Failed to remove staged sandbox write")
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
resolved_path = Path(self._resolve_path(path))
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results)
return [self._reverse_resolve_path(match) for match in matches], truncated
def grep(
self,
path: str,
pattern: str,
*,
glob: str | None = None,
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
) -> tuple[list[GrepMatch], bool]:
resolved_path = Path(self._resolve_path(path))
matches, truncated = find_grep_matches(
resolved_path,
pattern,
glob_pattern=glob,
literal=literal,
case_sensitive=case_sensitive,
max_results=max_results,
)
return [
GrepMatch(
path=self._reverse_resolve_path(match.path),
line_number=match.line_number,
line=match.line,
)
for match in matches
], truncated
def update_file(self, path: str, content: bytes) -> None:
resolved = self._resolve_path_with_mapping(path)
resolved_path = resolved.path
if self._is_resolved_path_read_only(resolved):
raise OSError(errno.EROFS, "Read-only file system", path)
try:
dir_path = os.path.dirname(resolved_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
with open(resolved_path, "wb") as f:
f.write(content)
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None