sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,15 +1,23 @@
import errno
import logging
import ntpath
import os
import shutil
import signal
import stat
import subprocess
import tempfile
import time
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import NamedTuple
from typing import BinaryIO, NamedTuple
from deerflow.sandbox.local.list_dir import list_dir
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
@@ -26,7 +34,40 @@ class ResolvedPath(NamedTuple):
mapping: PathMapping | None
class LocalSandbox(Sandbox):
class LocalSandbox(Sandbox):
_COMMAND_TIMEOUT_SECONDS = 600
_COMMAND_CLEANUP_SECONDS = 5
# Tool-level truncation happens after execution. Never first materialize an
# arbitrarily large command response in the Gateway process.
_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024
_COMMAND_CAPTURE_MAX_BYTES = 16 * 1024 * 1024
_MODEL_COMMAND_HIDDEN_ENV_KEYS = frozenset(
{
"DEER_FLOW_CONFIG_PATH",
"DEER_FLOW_EXTENSIONS_CONFIG_PATH",
"DEER_FLOW_HOME",
"DEER_FLOW_HOST_BASE_DIR",
"DEER_FLOW_HOST_SKILLS_PATH",
"DEER_FLOW_PROJECT_ROOT",
"DEER_FLOW_SKILLS_PATH",
"PYTHONPATH",
}
)
@staticmethod
def _model_command_environment() -> dict[str, str]:
"""Return a reduced environment for model-originated host commands.
LocalSandbox is deliberately not an OS security sandbox, but it should
not hand a model canonical skills/config paths through process
environment variables. ``PYTHONPATH`` is also removed so an editable
backend checkout is not advertised as an import root. Non-path
``DEER_FLOW_*`` settings (for example the Gateway port), ordinary PATH,
proxy and skill-specific credentials remain available.
"""
return {key: value for key, value in os.environ.items() if key.upper() not in LocalSandbox._MODEL_COMMAND_HIDDEN_ENV_KEYS}
@staticmethod
def _shell_name(shell: str) -> str:
"""Return the executable name for a shell path or command."""
@@ -37,26 +78,26 @@ class LocalSandbox(Sandbox):
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
@@ -246,7 +287,14 @@ class LocalSandbox(Sandbox):
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
return self._resolve_path(matched_path)
resolved = self._resolve_path(matched_path)
# Normalize to forward slashes: on Windows hosts the resolved path
# contains backslashes that the shell (bash -c) would treat as
# escape characters, corrupting the rewritten command (e.g.
# F:\repo\... losing its separators). Same normalization as
# ``_resolve_paths_in_content``; bash and Windows tools all accept
# forward-slash paths.
return resolved.replace("\\", "/")
return pattern.sub(replace_match, command)
@@ -314,48 +362,126 @@ class LocalSandbox(Sandbox):
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-NonInteractive",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
)
else:
args = [shell, "-c", resolved_command]
result = subprocess.run(
# Do not use PIPE + subprocess.run(timeout=...). Killing only the shell
# leaves skill subprocesses alive; on Windows run() then calls an
# unbounded communicate() while those children still hold the pipes.
# File-backed output also makes shell exit independent of a background
# child's inherited stdout/stderr and bounds Python memory usage.
timeout = self._command_timeout_seconds()
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
process = subprocess.Popen(
args,
shell=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=600,
stdin=subprocess.DEVNULL,
stdout=stdout,
stderr=stderr,
start_new_session=os.name != "nt",
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
env=self._model_command_environment(),
)
output = result.stdout
if result.stderr:
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
if result.returncode != 0:
output += f"\nExit Code: {result.returncode}"
failure: str | None = None
try:
deadline = time.monotonic() + timeout
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(args, timeout)
try:
process.wait(timeout=min(0.1, remaining))
break
except subprocess.TimeoutExpired:
# Avoid turning an unbounded in-memory PIPE into an
# unbounded temporary file (for example `yes`).
if os.fstat(stdout.fileno()).st_size + os.fstat(stderr.fileno()).st_size > self._COMMAND_CAPTURE_MAX_BYTES:
failure = f"Command output exceeded {self._COMMAND_CAPTURE_MAX_BYTES} bytes"
self._terminate_command_tree(process)
break
except subprocess.TimeoutExpired:
failure = f"Command timed out after {timeout:g} seconds"
self._terminate_command_tree(process)
except BaseException:
self._terminate_command_tree(process)
raise
output = self._read_command_output(stdout)
error_output = self._read_command_output(stderr)
if error_output:
output += f"\nStd Error:\n{error_output}" if output else error_output
if failure:
output = f"Error: {failure}; process tree termination requested.\n{output}"
elif process.returncode != 0:
output += f"\nExit Code: {process.returncode}"
final_output = output if output else "(no output)"
# Reverse resolve local paths back to container paths in output
return self._reverse_resolve_paths_in_output(final_output)
def _command_timeout_seconds(self) -> float:
try:
from deerflow.config import get_app_config
return get_app_config().sandbox.command_timeout_seconds
except Exception:
return self._COMMAND_TIMEOUT_SECONDS
@classmethod
def _read_command_output(cls, handle: BinaryIO) -> str:
size = handle.seek(0, os.SEEK_END)
limit = cls._COMMAND_OUTPUT_MAX_BYTES
handle.seek(0)
if size <= limit:
return handle.read(limit).decode("utf-8", errors="replace")
head = handle.read(limit // 2).decode("utf-8", errors="replace")
handle.seek(-limit // 2, os.SEEK_END)
tail = handle.read(limit // 2).decode("utf-8", errors="replace")
return f"{head}\n... [command output truncated; {size} bytes total] ...\n{tail}"
@classmethod
def _terminate_command_tree(cls, process: subprocess.Popen) -> None:
"""Kill descendants before reaping the shell, with bounded cleanup."""
try:
if os.name == "nt":
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=cls._COMMAND_CLEANUP_SECONDS,
check=False,
creationflags=subprocess.CREATE_NO_WINDOW,
)
else:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except (OSError, subprocess.TimeoutExpired):
logger.warning("Failed to terminate sandbox command process tree (pid=%s)", process.pid, exc_info=True)
finally:
if process.poll() is None:
try:
process.kill()
except ProcessLookupError:
pass
try:
process.wait(timeout=cls._COMMAND_CLEANUP_SECONDS)
except subprocess.TimeoutExpired:
logger.warning("Sandbox command did not exit after termination (pid=%s)", process.pid)
def list_dir(self, path: str, max_depth=2) -> list[str]:
resolved_path = self._resolve_path(path)
@@ -369,10 +495,10 @@ class LocalSandbox(Sandbox):
result.append(f"{reversed_entry}/" if is_dir and not reversed_entry.endswith("/") else reversed_entry)
return result
def read_file(self, path: str) -> str:
def read_file(self, path: str) -> str:
resolved_path = self._resolve_path(path)
try:
with open(resolved_path, encoding="utf-8") as f:
with self._open_regular_text(resolved_path) as f:
content = f.read()
# Only reverse-resolve paths in files that were previously written
# by write_file (agent-authored content). User-uploaded files,
@@ -383,7 +509,90 @@ class LocalSandbox(Sandbox):
return content
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
@contextmanager
def _open_regular_text(path: str, *, errors: str = "strict", newline: str | None = None):
# A skill script can create a FIFO inside its workspace. Opening it
# before checking the descriptor type would wait forever for a writer.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0))
try:
mode = os.fstat(fd).st_mode
if stat.S_ISDIR(mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory, not a file", path)
if not stat.S_ISREG(mode):
raise OSError(errno.EINVAL, "Only regular files can be read", path)
with os.fdopen(fd, "r", encoding="utf-8", errors=errors, newline=newline, closefd=False) as handle:
yield handle
finally:
os.close(fd)
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Stream a bounded range without materializing the whole file."""
resolved_path = self._resolve_path(path)
lines: list[str] = []
chars = 0
next_line: int | None = None
start_line = max(1, start_line)
try:
with self._open_regular_text(resolved_path, errors="replace", newline="") as handle:
number = 0
while True:
fragment = handle.readline(4097)
if not fragment:
break
number += 1
remaining = max_chars - chars - (1 if lines else 0)
capture = number >= start_line and (end_line is None or number <= end_line)
if end_line is not None and number > end_line:
break
if capture and remaining <= 0:
next_line = number
break
parts: list[str] = []
captured = 0
over_budget = False
while True:
ended = fragment.endswith(("\n", "\r"))
body = fragment.rstrip("\r\n") if ended else fragment
if capture and remaining > captured:
piece = body[: remaining - captured]
parts.append(piece)
captured += len(piece)
if len(piece) < len(body):
over_budget = True
elif capture and body:
over_budget = True
# The next request starts from the file's beginning,
# so continuation does not require draining the rest
# of an arbitrarily large physical line now.
if ended or (capture and captured >= remaining):
break
fragment = handle.readline(4097)
if not fragment:
break
if not capture:
continue
line = "".join(parts)
lines.append(line)
chars += len(line) + (1 if len(lines) > 1 else 0)
if over_budget or chars >= max_chars:
next_line = number + 1
break
content = "\n".join(lines)
if resolved_path in self._agent_written_paths:
content = self._reverse_resolve_paths_in_output(content)
return content, next_line, next_line is not None
except OSError as e:
raise type(e)(e.errno, e.strerror, path) from None
def write_file(self, path: str, content: str, append: bool = False) -> None:
resolved = self._resolve_path_with_mapping(path)
@@ -397,9 +606,11 @@ class LocalSandbox(Sandbox):
# Resolve container paths in content to local paths
# using the content-specific resolver (forward-slash safe)
resolved_content = self._resolve_paths_in_content(content)
mode = "a" if append else "w"
with open(resolved_path, mode, encoding="utf-8") as f:
f.write(resolved_content)
if append:
with open(resolved_path, "a", encoding="utf-8") as f:
f.write(resolved_content)
else:
self._atomic_overwrite(resolved_path, resolved_content)
# Track this path so read_file knows to reverse-resolve on read.
# Only agent-written files get reverse-resolved; user uploads and
# external tool output are left untouched.
@@ -408,6 +619,48 @@ class LocalSandbox(Sandbox):
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
@staticmethod
def _atomic_overwrite(path: str, content: str) -> None:
"""Publish a complete file only after encoding and disk writes succeed."""
# Preserve ordinary open(..., "w") symlink-following semantics.
target = os.path.realpath(path)
previous_mode: int | None = None
try:
previous_stat = os.stat(target)
except FileNotFoundError:
pass
else:
if stat.S_ISDIR(previous_stat.st_mode):
raise IsADirectoryError(errno.EISDIR, "Path is a directory", path)
if not stat.S_ISREG(previous_stat.st_mode):
raise OSError(errno.EINVAL, "Target is not a regular file", path)
# Replacement depends on directory permissions; also enforce the
# existing file's write permissions, without truncating it.
descriptor = os.open(target, os.O_WRONLY)
os.close(descriptor)
previous_mode = stat.S_IMODE(previous_stat.st_mode)
staged_path: str | None = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=os.path.dirname(target),
prefix=".deerflow-write-", suffix=".tmp", delete=False,
) as staged:
staged_path = staged.name
staged.write(content)
staged.flush()
os.fsync(staged.fileno())
if previous_mode is not None:
os.chmod(staged_path, previous_mode)
os.replace(staged_path, target)
staged_path = None
finally:
if staged_path is not None:
try:
os.unlink(staged_path)
except OSError:
logger.warning("Failed to remove staged sandbox write")
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
resolved_path = Path(self._resolve_path(path))
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results)
@@ -1,9 +1,10 @@
import logging
from pathlib import Path
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import SandboxProvider
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import SandboxProvider
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
logger = logging.getLogger(__name__)
@@ -29,13 +30,15 @@ class LocalSandboxProvider(SandboxProvider):
"""
mappings: list[PathMapping] = []
# Map skills container path to local skills directory
try:
from deerflow.config import get_app_config
config = get_app_config()
skills_path = config.skills.get_skills_path()
container_path = config.skills.container_path
# Map the model-facing path to the online-only materialized view. The
# canonical tree also contains offline skills and is administration
# state, not a sandbox capability.
try:
from deerflow.config import get_app_config
config = get_app_config()
skills_path = refresh_skill_sandbox_view().root
container_path = config.skills.container_path
# Only add mapping if skills directory exists
if skills_path.exists():
@@ -28,7 +28,7 @@ class Sandbox(ABC):
pass
@abstractmethod
def read_file(self, path: str) -> str:
def read_file(self, path: str) -> str:
"""Read the content of a file.
Args:
@@ -37,7 +37,38 @@ class Sandbox(ABC):
Returns:
The content of the file.
"""
pass
pass
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Read a bounded line range.
Providers should override this method to avoid loading the entire file.
The compatibility implementation remains bounded at the tool boundary,
though an older remote provider may still materialize its response.
"""
content = self.read_file(path)
selected: list[str] = []
chars = 0
next_line: int | None = None
for number, line in enumerate(content.splitlines(), start=1):
if number < start_line:
continue
if end_line is not None and number > end_line:
break
addition = len(line) + (1 if selected else 0)
if chars + addition > max_chars:
next_line = number
break
selected.append(line)
chars += addition
return "\n".join(selected), next_line, next_line is not None
@abstractmethod
def list_dir(self, path: str, max_depth=2) -> list[str]:
@@ -1,9 +1,10 @@
import fnmatch
import os
import re
import os
import re
import time
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
IGNORE_PATTERNS = [
".git",
".svn",
@@ -56,8 +57,10 @@ IGNORE_PATTERNS = [
".ruff_cache",
]
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
DEFAULT_LINE_SUMMARY_LENGTH = 200
DEFAULT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024
DEFAULT_MAX_TOTAL_SCAN_BYTES = 64 * 1024 * 1024
DEFAULT_MAX_SCAN_SECONDS = 5.0
DEFAULT_LINE_SUMMARY_LENGTH = 200
@dataclass(frozen=True)
@@ -148,7 +151,9 @@ def find_grep_matches(
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
max_file_size: int | None = None,
max_total_scan_bytes: int = DEFAULT_MAX_TOTAL_SCAN_BYTES,
max_scan_seconds: float = DEFAULT_MAX_SCAN_SECONDS,
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
) -> tuple[list[GrepMatch], bool]:
matches: list[GrepMatch] = []
@@ -162,16 +167,31 @@ def find_grep_matches(
regex_source = re.escape(pattern) if literal else pattern
flags = 0 if case_sensitive else re.IGNORECASE
regex = re.compile(regex_source, flags)
# Skip lines longer than this to prevent ReDoS on minified / no-newline files.
_max_line_chars = line_summary_length * 10
regex = re.compile(regex_source, flags)
if max_file_size is None:
try:
from deerflow.uploads.policy import get_runtime_upload_policy
groups = (get_runtime_upload_policy().get("groups") or {}).values()
configured = max((int(group.get("max_bytes") or 0) for group in groups), default=DEFAULT_MAX_FILE_SIZE_BYTES)
max_file_size = min(200 * 1024 * 1024, max(DEFAULT_MAX_FILE_SIZE_BYTES, configured))
except Exception:
max_file_size = DEFAULT_MAX_FILE_SIZE_BYTES
# Read lines in bounded fragments to prevent a minified / no-newline file
# from allocating its entire body before the length check can run.
_max_line_chars = line_summary_length * 10
deadline = time.monotonic() + max(0.1, max_scan_seconds)
scanned_bytes = 0
for current_root, dirs, files in os.walk(root):
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
rel_dir = Path(current_root).relative_to(root)
for name in files:
for name in files:
if time.monotonic() >= deadline:
return matches, True
if should_ignore_name(name):
continue
@@ -187,13 +207,32 @@ def find_grep_matches(
file_path = candidate_path.resolve()
if not file_path.is_relative_to(root):
continue
if file_path.stat().st_size > max_file_size or is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
for line_number, line in enumerate(handle, start=1):
if len(line) > _max_line_chars:
continue
if regex.search(line):
file_size = file_path.stat().st_size
if file_size > max_file_size:
truncated = True
continue
if scanned_bytes + file_size > max_total_scan_bytes:
return matches, True
scanned_bytes += file_size
if is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
line_number = 0
while True:
if time.monotonic() >= deadline:
return matches, True
line = handle.readline(_max_line_chars + 1)
if not line:
break
line_number += 1
if len(line) > _max_line_chars and not line.endswith(("\n", "\r")):
# Drain the rest of this physical line in bounded
# fragments, but never run an attacker-controlled
# regex over it.
while line and not line.endswith(("\n", "\r")):
line = handle.readline(_max_line_chars + 1)
continue
if regex.search(line):
matches.append(
GrepMatch(
path=str(file_path),
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,41 @@
"""A skill snapshot is reusable only inside one explicitly enabled tool call."""
from collections.abc import Callable
from contextvars import ContextVar
from dataclasses import dataclass
from functools import wraps
from typing import TYPE_CHECKING
from deerflow.config.runtime_performance_config import get_runtime_performance_config
if TYPE_CHECKING:
from deerflow.sandbox.skill_view import SkillSandboxView
@dataclass
class SkillViewScope:
view: "SkillSandboxView | None" = None
error: Exception | None = None
_scope: ContextVar[SkillViewScope | None] = ContextVar("sandbox_skill_view_scope", default=None)
def get_active_skill_view_scope() -> SkillViewScope | None:
return _scope.get()
def with_skill_view_scope[**P, R](func: Callable[P, R]) -> Callable[P, R]:
@wraps(func)
def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
if not get_runtime_performance_config().skill_view_reuse_enabled:
return func(*args, **kwargs)
token = _scope.set(SkillViewScope())
try:
return func(*args, **kwargs)
finally:
# Do not leak a previously enabled skill into the next tool call,
# including when a tool raises or shares an executor thread.
_scope.reset(token)
return wrapped
@@ -1,14 +1,21 @@
import json
import posixpath
import re
import shlex
from pathlib import Path
import time
from collections.abc import Callable
from functools import wraps
from inspect import signature
from pathlib import Path, PurePosixPath
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from pydantic import BaseModel, ConfigDict, Field
from deerflow.agents.thread_state import ThreadDataState, ThreadState
from deerflow.config import get_app_config
from deerflow.config.paths import VIRTUAL_PATH_PREFIX
from deerflow.html_runtime.html_postprocess import maybe_fix_generated_html
from deerflow.sandbox.exceptions import (
SandboxError,
SandboxNotFoundError,
@@ -19,6 +26,7 @@ from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
from deerflow.sandbox.search import GrepMatch
from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed
from deerflow.sandbox.skill_view_scope import get_active_skill_view_scope, with_skill_view_scope
_ABSOLUTE_PATH_PATTERN = re.compile(r"(?<![:\w])(?<!:/)/(?:[^\s\"'`;&|<>()]+)")
_FILE_URL_PATTERN = re.compile(r"\bfile://\S+", re.IGNORECASE)
@@ -99,36 +107,189 @@ def _get_skills_container_path() -> str:
def _get_skills_host_path() -> str | None:
"""Get the skills host filesystem path from config.
"""Get the online-only model-facing skills path.
Returns None if the skills directory does not exist or config cannot be
loaded. Only successful lookups are cached; failures are retried on the
next call so that a transiently unavailable skills directory does not
permanently disable skills access.
Never return the canonical administrative skills tree. Offline skills
intentionally remain there and would otherwise be reachable from bash by
constructing a path dynamically instead of spelling it in the tool call.
A warm cache resolves the published view without stating every skill file.
"""
cached = getattr(_get_skills_host_path, "_cached", None)
if cached is not None:
return cached
try:
from deerflow.config import get_app_config
from deerflow.sandbox.skill_view import current_skill_view_root
config = get_app_config()
skills_path = config.skills.get_skills_path()
if skills_path.exists():
value = str(skills_path)
_get_skills_host_path._cached = value # type: ignore[attr-defined]
return value
return str(current_skill_view_root())
except Exception:
pass
return None
def _refresh_skill_capability() -> None:
"""Refresh the filesystem capability used by local and AIO sandboxes."""
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
refresh_skill_sandbox_view()
def _prepare_skill_paths(paths: list[str]) -> None:
"""Refresh only the skills a tool is about to read, when the cache is on."""
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
if not _skill_view_cache_enabled():
_refresh_skill_capability()
return
prepare_skill_tool_view(paths, container_path=_get_skills_container_path())
def _prepare_command_skill_capability(command: str) -> None:
"""Skip the skill-view check when a command does not name a skill.
The online view is still the security boundary. Management writes remove
an offlined skill from that view immediately, so a later command that
hides the path cannot run it. Commands that spell a skill path reconcile
only that skill.
"""
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
if not _skill_view_cache_enabled():
_refresh_skill_capability()
return
skills_container = _get_skills_container_path()
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
if boundary.search(command) is None:
from deerflow.sandbox.skill_view import note_skill_view_check
note_skill_view_check(status="skipped")
return
paths: list[str] = []
url_spans = _non_file_url_spans(command)
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
if _is_in_spans(match.start(), url_spans):
continue
path = match.group()
if _is_skills_path(path):
paths.append(path)
prepare_skill_tool_view(paths, container_path=skills_container)
def _skill_paths_in_text(text: str) -> list[str]:
paths: list[str] = []
for match in _ABSOLUTE_PATH_PATTERN.finditer(text):
path = match.group()
if _is_skills_path(path) and path not in paths:
paths.append(path)
return paths
def _finish_skill_view_trace(tool: str, command: str, started: float, *, always: bool) -> None:
"""Write the diagnostic row only after this tool call has returned."""
try:
from deerflow.sandbox.skill_view import commit_skill_view_trace, skill_names_for_paths
commit_skill_view_trace(
tool=tool,
command=command,
skill_names=skill_names_for_paths(_skill_paths_in_text(command), container_path=_get_skills_container_path()),
tool_ms=(time.perf_counter() - started) * 1000,
allow_empty=always,
)
except Exception:
return
def _trace_completed_tool(tool_name: str, *, command_from: str, always: bool = False) -> Callable:
"""Publish one timing row after the wrapped tool returns, including on error."""
def decorator[**P, R](fn: Callable[P, R]) -> Callable[P, R]:
sig = signature(fn)
@wraps(fn)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
from deerflow.sandbox.skill_view import begin_skill_view_trace, end_skill_view_trace
bound = sig.bind_partial(*args, **kwargs)
command = str(bound.arguments.get(command_from) or "")
if tool_name in {"glob", "grep"}:
command = f"{bound.arguments.get('pattern') or ''} {command}".strip()
started = time.perf_counter()
token = begin_skill_view_trace()
try:
return fn(*args, **kwargs)
finally:
try:
_finish_skill_view_trace(tool_name, command, started, always=always)
finally:
end_skill_view_trace(token)
wrapper.__signature__ = sig # type: ignore[attr-defined]
return wrapper
return decorator
def _is_skills_path(path: str) -> bool:
"""Check if a path is under the skills container path."""
skills_prefix = _get_skills_container_path()
return path == skills_prefix or path.startswith(f"{skills_prefix}/")
def _require_enabled_skill_path(path: str) -> None:
"""Fail closed unless ``path`` belongs to a currently enabled skill."""
scope = get_active_skill_view_scope()
if scope is not None:
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
# The entry snapshot already validated availability and discovered the
# real skill directories. Re-parsing the full catalogue here (and again
# during local path validation) would defeat per-call view reuse.
view = refresh_skill_sandbox_view()
prefix = _get_skills_container_path().replace("\\", "/").rstrip("/")
normalized = path.replace("\\", "/").rstrip("/")
if ".." in PurePosixPath(normalized).parts:
raise PermissionError("Skill path traversal is not allowed")
for _name, category, relative in view.enabled_roots:
# Root/category-wide access is never a model capability.
if not relative or relative == ".":
continue
skill_root = f"{prefix}/{category}/{relative}"
if normalized == skill_root or normalized.startswith(f"{skill_root}/"):
return
raise PermissionError("Skill path is unavailable")
from deerflow.tools.builtins.skill_tools import require_enabled_skill_path
require_enabled_skill_path(path, container_path=_get_skills_container_path())
def _validate_enabled_skill_paths_in_command(command: str) -> None:
"""Give a clear error for explicit unavailable skill paths.
This is a usability policy, not the security boundary. Shell expansion
can hide paths from string inspection, so the enforceable boundary is the
online-only filesystem view. Management writes update that view immediately;
a command that names a skill reconciles that skill before it runs.
"""
skills_container = _get_skills_container_path()
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
if boundary.search(command) is None:
return
matched = False
url_spans = _non_file_url_spans(command)
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
if _is_in_spans(match.start(), url_spans):
continue
path = match.group()
if not _is_skills_path(path):
continue
matched = True
_require_enabled_skill_path(path)
# Quoting/concatenation that hides the concrete skill directory is not a
# valid way around the gate. Exact, discoverable paths are required.
if not matched:
raise PermissionError("Skill path is unavailable")
def _resolve_skills_path(path: str) -> str:
"""Resolve a virtual skills path to a host filesystem path.
@@ -649,6 +810,7 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *,
# Skills paths — read-only access only
if _is_skills_path(path):
_require_enabled_skill_path(path)
if not read_only:
raise PermissionError(f"Write access to skills path is not allowed: {path}")
return
@@ -1268,6 +1430,8 @@ def _truncate_ls_output(output: str, max_chars: int) -> str:
@tool("bash", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("bash", command_from="command", always=True)
def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str:
"""Execute a bash command in a Linux environment.
@@ -1281,6 +1445,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
command: The bash command to execute. Always use absolute paths for files and directories.
"""
try:
_prepare_command_skill_capability(command)
_validate_enabled_skill_paths_in_command(command)
sandbox = ensure_sandbox_initialized(runtime)
if is_local_sandbox(runtime):
if not is_host_bash_allowed():
@@ -1317,6 +1483,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
@tool("ls", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("ls", command_from="path")
def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: str) -> str:
"""List the contents of a directory up to 2 levels deep in tree format.
@@ -1324,10 +1492,13 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
description: Explain why you are listing this directory in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
path: The **absolute** path to the directory to list.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
thread_data = None
if is_local_sandbox(runtime):
thread_data = get_thread_data(runtime)
@@ -1364,6 +1535,8 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
@tool("glob", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("glob", command_from="path")
def glob_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1381,10 +1554,13 @@ def glob_tool(
include_dirs: Whether matching directories should also be returned. Default is False.
max_results: Maximum number of paths to return. Default is 200.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"glob",
max_results,
@@ -1414,6 +1590,8 @@ def glob_tool(
@tool("grep", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("grep", command_from="path")
def grep_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1435,10 +1613,13 @@ def grep_tool(
case_sensitive: Whether matching is case-sensitive. Default is False.
max_results: Maximum number of matching lines to return. Default is 100.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"grep",
max_results,
@@ -1484,6 +1665,8 @@ def grep_tool(
@tool("read_file", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("read_file", command_from="path")
def read_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1501,15 +1684,27 @@ def read_file_tool(
"""
try:
requested_path = path
if start_line is not None and (not isinstance(start_line, int) or isinstance(start_line, bool) or start_line < 1):
return "Error: start_line must be a positive integer"
if end_line is not None and (not isinstance(end_line, int) or isinstance(end_line, bool) or end_line < 1):
return "Error: end_line must be a positive integer"
effective_start = start_line or 1
if end_line is not None and end_line < effective_start:
return "Error: end_line must be greater than or equal to start_line"
if _is_skills_path(requested_path):
_prepare_skill_paths([requested_path])
from deerflow.tools.builtins.skill_tools import (
_resolve_active_agent_id,
can_access_coordinator_only_skill,
is_coordinator_only_skill_path,
)
if is_coordinator_only_skill_path(requested_path) and _resolve_active_agent_id(runtime) != "roundtable-coordinator":
if is_coordinator_only_skill_path(requested_path) and not can_access_coordinator_only_skill(
_resolve_active_agent_id(runtime), "agent_orchestration"
):
# Match skill_view's non-leaking behavior: ordinary agents must not discover whether it exists.
return f"Error: File not found: {requested_path}"
_require_enabled_skill_path(requested_path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
if is_local_sandbox(runtime):
@@ -1522,21 +1717,62 @@ def read_file_tool(
elif not _is_custom_mount_path(path):
path = _resolve_and_validate_user_data_path(path, thread_data)
# Custom mount paths are resolved by LocalSandbox._resolve_path()
content = sandbox.read_file(path)
try:
from deerflow.config.app_config import get_app_config
from deerflow.uploads.policy import get_runtime_large_file_policy
sandbox_cfg = get_app_config().sandbox
configured_max = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
large_file_policy = get_runtime_large_file_policy()
except Exception:
configured_max = 50000
large_file_policy = {
"threshold_bytes": 1024 * 1024,
"max_result_chars": 12000,
"hard_max_result_chars": 32000,
}
policy_max = max(256, int(large_file_policy.get("max_result_chars", 12000)))
hard_max = max(256, min(32000, int(large_file_policy.get("hard_max_result_chars", 32000))))
max_chars = min(policy_max, hard_max, configured_max if configured_max > 0 else hard_max)
# A range-free read of a large local file returns deterministic
# head/middle/tail samples plus whole-file streaming statistics. The
# raw body is never materialized in memory or sent to the model.
if start_line is None and end_line is None and is_local_sandbox(runtime):
try:
physical_path = Path(path)
threshold = max(1, int(large_file_policy.get("threshold_bytes", 1024 * 1024)))
if physical_path.is_file() and physical_path.stat().st_size >= threshold:
from deerflow.uploads.inspection import inspect_text_file
inspected = inspect_text_file(
physical_path,
display_path=requested_path,
max_result_chars=max_chars,
hard_max_result_chars=hard_max,
)
_record_skill_read(requested_path)
return inspected["rendered"]
except OSError:
pass
# Reserve room for a continuation marker so the complete tool result,
# not just its body, remains under the hard cap.
body_budget = max(64, max_chars - 180)
content, next_line, truncated = sandbox.read_file_range(
path,
start_line=effective_start,
end_line=end_line,
max_chars=body_budget,
)
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
# 非技能路径会被 _record_skill_read 自行忽略。
_record_skill_read(requested_path)
if not content:
if not content and not truncated:
return "(empty)"
if start_line is not None and end_line is not None:
content = "\n".join(content.splitlines()[start_line - 1 : end_line])
try:
from deerflow.config.app_config import get_app_config
sandbox_cfg = get_app_config().sandbox
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
except Exception:
max_chars = 50000
if truncated:
continuation = next_line or effective_start
content = f"{content}\n... [bounded read; continue with start_line={continuation}]"
return _truncate_read_file_output(content, max_chars)
except SandboxError as e:
return f"Error: {e}"
@@ -1551,11 +1787,12 @@ def read_file_tool(
@tool("write_file", parse_docstring=True)
@with_skill_view_scope
def write_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
path: str,
content: str,
description: str = "",
path: str = "",
content: str = "",
append: bool = False,
) -> str:
"""Write text content to a file.
@@ -1565,6 +1802,13 @@ def write_file_tool(
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
content: The content to write to the file. ALWAYS PROVIDE THIS PARAMETER THIRD.
"""
if not str(path).strip():
return "Error: write_file requires an absolute path."
if content is None or content == "":
return (
"Error: write_file requires the full file content in the same call as path. "
"Retry once with path and content together."
)
try:
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
@@ -1576,6 +1820,7 @@ def write_file_tool(
path = _resolve_and_validate_user_data_path(path, thread_data)
# Custom mount paths are resolved by LocalSandbox._resolve_path()
with get_file_operation_lock(sandbox, path):
content = maybe_fix_generated_html(path, content)
sandbox.write_file(path, content, append)
return "OK"
except SandboxError as e:
@@ -1590,7 +1835,90 @@ def write_file_tool(
return f"Error: Unexpected error writing file: {_sanitize_error(e, runtime)}"
class _BeginWriteFileInput(BaseModel):
"""Small, strict prepare envelope; file bytes are never valid here."""
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
runtime: ToolRuntime[ContextT, ThreadState]
description: str = Field(
default="",
max_length=240,
description="A short explanation of the file being written.",
)
path: str = Field(
min_length=1,
max_length=4096,
description="The absolute sandbox path of the target file.",
)
append: bool = Field(
default=False,
description="Whether the final write appends instead of overwriting.",
)
@tool(
"begin_write_file",
args_schema=_BeginWriteFileInput,
parse_docstring=True,
)
@with_skill_view_scope
def begin_write_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str = "",
path: str = "",
append: bool = False,
) -> str:
"""Prepare a file target for a streamed text-body write without writing it yet.
This internal compatibility handshake is exposed only to models configured
with ``write_file_transport: artifact_renderer`` (or the legacy
``text_bridge`` mode). It validates the target but writes no bytes. After a
``status=ready`` result, DeerFlow obtains the body through the configured
transport and executes the real ``write_file`` tool only after validation.
Args:
description: Explain why you are writing to this file in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
append: Whether the final write should append instead of overwrite.
"""
if not str(path).strip().startswith("/"):
return "Error: begin_write_file requires an absolute path."
requested_path = path
try:
# Match write_file's sandbox acquisition and local path authorization,
# but deliberately do not take a file lock or persist any bytes.
ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
if is_local_sandbox(runtime):
thread_data = get_thread_data(runtime)
validate_local_tool_path(path, thread_data)
if not _is_custom_mount_path(path):
_resolve_and_validate_user_data_path(path, thread_data)
return json.dumps(
{
"status": "ready",
"path": requested_path,
"description": str(description or ""),
"append": append,
},
ensure_ascii=False,
separators=(",", ":"),
)
except SandboxError as e:
return f"Error: {e}"
except PermissionError:
return f"Error: Permission denied writing to file: {requested_path}"
except IsADirectoryError:
return f"Error: Path is a directory, not a file: {requested_path}"
except OSError as e:
return f"Error: Failed to prepare file '{requested_path}': {_sanitize_error(e, runtime)}"
except Exception as e:
return f"Error: Unexpected error preparing file: {_sanitize_error(e, runtime)}"
@tool("str_replace", parse_docstring=True)
@with_skill_view_scope
def str_replace_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1629,6 +1957,7 @@ def str_replace_tool(
content = content.replace(old_str, new_str)
else:
content = content.replace(old_str, new_str, 1)
content = maybe_fix_generated_html(path, content)
sandbox.write_file(path, content)
return "OK"
except SandboxError as e: