sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,15 +1,23 @@
import errno
import logging
import ntpath
import os
import shutil
import signal
import stat
import subprocess
import tempfile
import time
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import NamedTuple
from typing import BinaryIO, NamedTuple
from deerflow.sandbox.local.list_dir import list_dir
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@@ -26,7 +34,40 @@ class ResolvedPath(NamedTuple):
mapping: PathMapping | None
class LocalSandbox(Sandbox):
class LocalSandbox(Sandbox):
_COMMAND_TIMEOUT_SECONDS = 600
_COMMAND_CLEANUP_SECONDS = 5
# Tool-level truncation happens after execution. Never first materialize an
# arbitrarily large command response in the Gateway process.
_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024
_COMMAND_CAPTURE_MAX_BYTES = 16 * 1024 * 1024
_MODEL_COMMAND_HIDDEN_ENV_KEYS = frozenset(
{
"DEER_FLOW_CONFIG_PATH",
"DEER_FLOW_EXTENSIONS_CONFIG_PATH",
"DEER_FLOW_HOME",
"DEER_FLOW_HOST_BASE_DIR",
"DEER_FLOW_HOST_SKILLS_PATH",
"DEER_FLOW_PROJECT_ROOT",
"DEER_FLOW_SKILLS_PATH",
"PYTHONPATH",
}
)
@staticmethod
def _model_command_environment() -> dict[str, str]:
"""Return a reduced environment for model-originated host commands.
LocalSandbox is deliberately not an OS security sandbox, but it should
not hand a model canonical skills/config paths through process
environment variables. ``PYTHONPATH`` is also removed so an editable
backend checkout is not advertised as an import root. Non-path
``DEER_FLOW_*`` settings (for example the Gateway port), ordinary PATH,
proxy and skill-specific credentials remain available.
"""
return {key: value for key, value in os.environ.items() if key.upper() not in LocalSandbox._MODEL_COMMAND_HIDDEN_ENV_KEYS}
@staticmethod
def _shell_name(shell: str) -> str:
"""Return the executable name for a shell path or command."""
@@ -37,26 +78,26 @@ class LocalSandbox(Sandbox):
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
@@ -246,7 +287,14 @@ class LocalSandbox(Sandbox):
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
return self._resolve_path(matched_path)
resolved = self._resolve_path(matched_path)
# Normalize to forward slashes: on Windows hosts the resolved path
# contains backslashes that the shell (bash -c) would treat as
# escape characters, corrupting the rewritten command (e.g.
# F:\repo\... losing its separators). Same normalization as
# ``_resolve_paths_in_content``; bash and Windows tools all accept
# forward-slash paths.
return resolved.replace("\\", "/")
return pattern.sub(replace_match, command)
@@ -314,48 +362,126 @@ class LocalSandbox(Sandbox):
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-NonInteractive",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
# Do not use PIPE + subprocess.run(timeout=...). Killing only the shell
# leaves skill subprocesses alive; on Windows run() then calls an
# unbounded communicate() while those children still hold the pipes.
# File-backed output also makes shell exit independent of a background
# child's inherited stdout/stderr and bounds Python memory usage.
timeout = self._command_timeout_seconds()
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
process = subprocess.Popen(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
stdin=subprocess.DEVNULL,
stdout=stdout,
stderr=stderr,
start_new_session=os.name != "nt",
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
env=self._model_command_environment(),
)
output = result.stdout
if result.stderr:
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
if result.returncode != 0:
output += f"\nExit Code: {result.returncode}"
failure: str | None = None
try:
deadline = time.monotonic() + timeout
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(args, timeout)
try:
process.wait(timeout=min(0.1, remaining))
break
except subprocess.TimeoutExpired:
# Avoid turning an unbounded in-memory PIPE into an
# unbounded temporary file (for example `yes`).
if os.fstat(stdout.fileno()).st_size + os.fstat(stderr.fileno()).st_size > self._COMMAND_CAPTURE_MAX_BYTES:
failure = f"Command output exceeded {self._COMMAND_CAPTURE_MAX_BYTES} bytes"
self._terminate_command_tree(process)
break
except subprocess.TimeoutExpired:
failure = f"Command timed out after {timeout:g} seconds"
self._terminate_command_tree(process)
except BaseException:
self._terminate_command_tree(process)
raise
output = self._read_command_output(stdout)
error_output = self._read_command_output(stderr)
if error_output:
output += f"\nStd Error:\n{error_output}" if output else error_output
if failure:
output = f"Error: {failure}; process tree termination requested.\n{output}"
elif process.returncode != 0:
output += f"\nExit Code: {process.returncode}"
final_output = output if output else "(no output)"
# Reverse resolve local paths back to container paths in output
return self._reverse_resolve_paths_in_output(final_output)
def _command_timeout_seconds(self) -> float:
try:
from deerflow.config import get_app_config
return get_app_config().sandbox.command_timeout_seconds
except Exception:
return self._COMMAND_TIMEOUT_SECONDS
@classmethod
def _read_command_output(cls, handle: BinaryIO) -> str:
size = handle.seek(0, os.SEEK_END)
limit = cls._COMMAND_OUTPUT_MAX_BYTES
handle.seek(0)
if size <= limit:
return handle.read(limit).decode("utf-8", errors="replace")
head = handle.read(limit // 2).decode("utf-8", errors="replace")
handle.seek(-limit // 2, os.SEEK_END)
tail = handle.read(limit // 2).decode("utf-8", errors="replace")
return f"{head}\n... [command output truncated; {size} bytes total] ...\n{tail}"
@classmethod
def _terminate_command_tree(cls, process: subprocess.Popen) -> None:
"""Kill descendants before reaping the shell, with bounded cleanup."""
try:
if os.name == "nt":
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=cls._COMMAND_CLEANUP_SECONDS,
check=False,
creationflags=subprocess.CREATE_NO_WINDOW,
)
else:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except (OSError, subprocess.TimeoutExpired):
logger.warning("Failed to terminate sandbox command process tree (pid=%s)", process.pid, exc_info=True)
finally:
if process.poll() is None:
try:
process.kill()
except ProcessLookupError:
pass
try:
process.wait(timeout=cls._COMMAND_CLEANUP_SECONDS)
except subprocess.TimeoutExpired:
logger.warning("Sandbox command did not exit after termination (pid=%s)", process.pid)
def list_dir(self, path: str, max_depth=2) -> list[str]:
resolved_path = self._resolve_path(path)
@@ -369,10 +495,10 @@ class LocalSandbox(Sandbox):
result.append(f"{reversed_entry}/" if is_dir and not reversed_entry.endswith("/") else reversed_entry)
return result
def read_file(self, path: str) -> str:
def read_file(self, path: str) -> str:
resolved_path = self._resolve_path(path)
try:
with open(resolved_path, encoding="utf-8") as f:
with self._open_regular_text(resolved_path) as f:
content = f.read()
# Only reverse-resolve paths in files that were previously written
# by write_file (agent-authored content). User-uploaded files,
@@ -383,7 +509,90 @@ class LocalSandbox(Sandbox):
return content
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
@contextmanager
def _open_regular_text(path: str, *, errors: str = "strict", newline: str | None = None):
# A skill script can create a FIFO inside its workspace. Opening it
# before checking the descriptor type would wait forever for a writer.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0))
try:
mode = os.fstat(fd).st_mode
if stat.S_ISDIR(mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory, not a file", path)
if not stat.S_ISREG(mode):
raise OSError(errno.EINVAL, "Only regular files can be read", path)
with os.fdopen(fd, "r", encoding="utf-8", errors=errors, newline=newline, closefd=False) as handle:
yield handle
finally:
os.close(fd)
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Stream a bounded range without materializing the whole file."""
resolved_path = self._resolve_path(path)
lines: list[str] = []
chars = 0
next_line: int | None = None
start_line = max(1, start_line)
try:
with self._open_regular_text(resolved_path, errors="replace", newline="") as handle:
number = 0
while True:
fragment = handle.readline(4097)
if not fragment:
break
number += 1
remaining = max_chars - chars - (1 if lines else 0)
capture = number >= start_line and (end_line is None or number <= end_line)
if end_line is not None and number > end_line:
break
if capture and remaining <= 0:
next_line = number
break
parts: list[str] = []
captured = 0
over_budget = False
while True:
ended = fragment.endswith(("\n", "\r"))
body = fragment.rstrip("\r\n") if ended else fragment
if capture and remaining > captured:
piece = body[: remaining - captured]
parts.append(piece)
captured += len(piece)
if len(piece) < len(body):
over_budget = True
elif capture and body:
over_budget = True
# The next request starts from the file's beginning,
# so continuation does not require draining the rest
# of an arbitrarily large physical line now.
if ended or (capture and captured >= remaining):
break
fragment = handle.readline(4097)
if not fragment:
break
if not capture:
continue
line = "".join(parts)
lines.append(line)
chars += len(line) + (1 if len(lines) > 1 else 0)
if over_budget or chars >= max_chars:
next_line = number + 1
break
content = "\n".join(lines)
if resolved_path in self._agent_written_paths:
content = self._reverse_resolve_paths_in_output(content)
return content, next_line, next_line is not None
except OSError as e:
raise type(e)(e.errno, e.strerror, path) from None
def write_file(self, path: str, content: str, append: bool = False) -> None:
resolved = self._resolve_path_with_mapping(path)
@@ -397,9 +606,11 @@ class LocalSandbox(Sandbox):
# Resolve container paths in content to local paths
# using the content-specific resolver (forward-slash safe)
resolved_content = self._resolve_paths_in_content(content)
mode = "a" if append else "w"
with open(resolved_path, mode, encoding="utf-8") as f:
f.write(resolved_content)
if append:
with open(resolved_path, "a", encoding="utf-8") as f:
f.write(resolved_content)
else:
self._atomic_overwrite(resolved_path, resolved_content)
# Track this path so read_file knows to reverse-resolve on read.
# Only agent-written files get reverse-resolved; user uploads and
# external tool output are left untouched.
@@ -408,6 +619,48 @@ class LocalSandbox(Sandbox):
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
def _atomic_overwrite(path: str, content: str) -> None:
"""Publish a complete file only after encoding and disk writes succeed."""
# Preserve ordinary open(..., "w") symlink-following semantics.
target = os.path.realpath(path)
previous_mode: int | None = None
try:
previous_stat = os.stat(target)
except FileNotFoundError:
pass
else:
if stat.S_ISDIR(previous_stat.st_mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory", path)
if not stat.S_ISREG(previous_stat.st_mode):
raise OSError(errno.EINVAL, "Target is not a regular file", path)
# Replacement depends on directory permissions; also enforce the
# existing file's write permissions, without truncating it.
descriptor = os.open(target, os.O_WRONLY)
os.close(descriptor)
previous_mode = stat.S_IMODE(previous_stat.st_mode)
staged_path: str | None = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=os.path.dirname(target),
prefix=".deerflow-write-", suffix=".tmp", delete=False,
) as staged:
staged_path = staged.name
staged.write(content)
staged.flush()
os.fsync(staged.fileno())
if previous_mode is not None:
os.chmod(staged_path, previous_mode)
os.replace(staged_path, target)
staged_path = None
finally:
if staged_path is not None:
try:
os.unlink(staged_path)
except OSError:
logger.warning("Failed to remove staged sandbox write")
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
resolved_path = Path(self._resolve_path(path))
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results)