sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+310
-57
@@ -1,15 +1,23 @@
|
||||
import errno
|
||||
import logging
|
||||
import ntpath
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
from typing import BinaryIO, NamedTuple
|
||||
|
||||
from deerflow.sandbox.local.list_dir import list_dir
|
||||
from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.search import GrepMatch, find_glob_matches, find_grep_matches
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -26,7 +34,40 @@ class ResolvedPath(NamedTuple):
|
||||
mapping: PathMapping | None
|
||||
|
||||
|
||||
class LocalSandbox(Sandbox):
|
||||
class LocalSandbox(Sandbox):
|
||||
_COMMAND_TIMEOUT_SECONDS = 600
|
||||
_COMMAND_CLEANUP_SECONDS = 5
|
||||
# Tool-level truncation happens after execution. Never first materialize an
|
||||
# arbitrarily large command response in the Gateway process.
|
||||
_COMMAND_OUTPUT_MAX_BYTES = 1024 * 1024
|
||||
_COMMAND_CAPTURE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
_MODEL_COMMAND_HIDDEN_ENV_KEYS = frozenset(
|
||||
{
|
||||
"DEER_FLOW_CONFIG_PATH",
|
||||
"DEER_FLOW_EXTENSIONS_CONFIG_PATH",
|
||||
"DEER_FLOW_HOME",
|
||||
"DEER_FLOW_HOST_BASE_DIR",
|
||||
"DEER_FLOW_HOST_SKILLS_PATH",
|
||||
"DEER_FLOW_PROJECT_ROOT",
|
||||
"DEER_FLOW_SKILLS_PATH",
|
||||
"PYTHONPATH",
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _model_command_environment() -> dict[str, str]:
|
||||
"""Return a reduced environment for model-originated host commands.
|
||||
|
||||
LocalSandbox is deliberately not an OS security sandbox, but it should
|
||||
not hand a model canonical skills/config paths through process
|
||||
environment variables. ``PYTHONPATH`` is also removed so an editable
|
||||
backend checkout is not advertised as an import root. Non-path
|
||||
``DEER_FLOW_*`` settings (for example the Gateway port), ordinary PATH,
|
||||
proxy and skill-specific credentials remain available.
|
||||
"""
|
||||
return {key: value for key, value in os.environ.items() if key.upper() not in LocalSandbox._MODEL_COMMAND_HIDDEN_ENV_KEYS}
|
||||
|
||||
@staticmethod
|
||||
def _shell_name(shell: str) -> str:
|
||||
"""Return the executable name for a shell path or command."""
|
||||
@@ -37,26 +78,26 @@ class LocalSandbox(Sandbox):
|
||||
"""Return whether the selected shell is a PowerShell executable."""
|
||||
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _is_cmd_shell(shell: str) -> bool:
|
||||
"""Return whether the selected shell is cmd.exe."""
|
||||
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_powershell_command(command: str) -> str:
|
||||
"""Make bash-style command glue usable in Windows PowerShell 5.
|
||||
|
||||
The public tool is still named ``bash`` for model compatibility, but on
|
||||
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
|
||||
not support ``&&``, while the sandbox layer prepends commands with
|
||||
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
|
||||
executable in that fallback shell.
|
||||
"""
|
||||
return command.replace("&&", ";")
|
||||
|
||||
@staticmethod
|
||||
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
|
||||
"""Return the first executable shell path or command found from candidates."""
|
||||
@staticmethod
|
||||
def _is_cmd_shell(shell: str) -> bool:
|
||||
"""Return whether the selected shell is cmd.exe."""
|
||||
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_powershell_command(command: str) -> str:
|
||||
"""Make bash-style command glue usable in Windows PowerShell 5.
|
||||
|
||||
The public tool is still named ``bash`` for model compatibility, but on
|
||||
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
|
||||
not support ``&&``, while the sandbox layer prepends commands with
|
||||
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
|
||||
executable in that fallback shell.
|
||||
"""
|
||||
return command.replace("&&", ";")
|
||||
|
||||
@staticmethod
|
||||
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
|
||||
"""Return the first executable shell path or command found from candidates."""
|
||||
for shell in candidates:
|
||||
if os.path.isabs(shell):
|
||||
if os.path.isfile(shell) and os.access(shell, os.X_OK):
|
||||
@@ -246,7 +287,14 @@ class LocalSandbox(Sandbox):
|
||||
|
||||
def replace_match(match: re.Match) -> str:
|
||||
matched_path = match.group(0)
|
||||
return self._resolve_path(matched_path)
|
||||
resolved = self._resolve_path(matched_path)
|
||||
# Normalize to forward slashes: on Windows hosts the resolved path
|
||||
# contains backslashes that the shell (bash -c) would treat as
|
||||
# escape characters, corrupting the rewritten command (e.g.
|
||||
# F:\repo\... losing its separators). Same normalization as
|
||||
# ``_resolve_paths_in_content``; bash and Windows tools all accept
|
||||
# forward-slash paths.
|
||||
return resolved.replace("\\", "/")
|
||||
|
||||
return pattern.sub(replace_match, command)
|
||||
|
||||
@@ -314,48 +362,126 @@ class LocalSandbox(Sandbox):
|
||||
resolved_command = self._resolve_paths_in_command(command)
|
||||
shell = self._get_shell()
|
||||
|
||||
if os.name == "nt":
|
||||
if self._is_powershell(shell):
|
||||
args = [
|
||||
shell,
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
self._normalize_powershell_command(resolved_command),
|
||||
]
|
||||
if os.name == "nt":
|
||||
if self._is_powershell(shell):
|
||||
args = [
|
||||
shell,
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
self._normalize_powershell_command(resolved_command),
|
||||
]
|
||||
elif self._is_cmd_shell(shell):
|
||||
args = [shell, "/c", resolved_command]
|
||||
else:
|
||||
args = [shell, "-c", resolved_command]
|
||||
|
||||
result = subprocess.run(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
args = [shell, "-c", resolved_command]
|
||||
result = subprocess.run(
|
||||
|
||||
# Do not use PIPE + subprocess.run(timeout=...). Killing only the shell
|
||||
# leaves skill subprocesses alive; on Windows run() then calls an
|
||||
# unbounded communicate() while those children still hold the pipes.
|
||||
# File-backed output also makes shell exit independent of a background
|
||||
# child's inherited stdout/stderr and bounds Python memory usage.
|
||||
timeout = self._command_timeout_seconds()
|
||||
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
|
||||
process = subprocess.Popen(
|
||||
args,
|
||||
shell=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=600,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
start_new_session=os.name != "nt",
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||
env=self._model_command_environment(),
|
||||
)
|
||||
output = result.stdout
|
||||
if result.stderr:
|
||||
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
|
||||
if result.returncode != 0:
|
||||
output += f"\nExit Code: {result.returncode}"
|
||||
failure: str | None = None
|
||||
try:
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise subprocess.TimeoutExpired(args, timeout)
|
||||
try:
|
||||
process.wait(timeout=min(0.1, remaining))
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
# Avoid turning an unbounded in-memory PIPE into an
|
||||
# unbounded temporary file (for example `yes`).
|
||||
if os.fstat(stdout.fileno()).st_size + os.fstat(stderr.fileno()).st_size > self._COMMAND_CAPTURE_MAX_BYTES:
|
||||
failure = f"Command output exceeded {self._COMMAND_CAPTURE_MAX_BYTES} bytes"
|
||||
self._terminate_command_tree(process)
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
failure = f"Command timed out after {timeout:g} seconds"
|
||||
self._terminate_command_tree(process)
|
||||
except BaseException:
|
||||
self._terminate_command_tree(process)
|
||||
raise
|
||||
output = self._read_command_output(stdout)
|
||||
error_output = self._read_command_output(stderr)
|
||||
if error_output:
|
||||
output += f"\nStd Error:\n{error_output}" if output else error_output
|
||||
if failure:
|
||||
output = f"Error: {failure}; process tree termination requested.\n{output}"
|
||||
elif process.returncode != 0:
|
||||
output += f"\nExit Code: {process.returncode}"
|
||||
|
||||
final_output = output if output else "(no output)"
|
||||
# Reverse resolve local paths back to container paths in output
|
||||
return self._reverse_resolve_paths_in_output(final_output)
|
||||
|
||||
def _command_timeout_seconds(self) -> float:
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
return get_app_config().sandbox.command_timeout_seconds
|
||||
except Exception:
|
||||
return self._COMMAND_TIMEOUT_SECONDS
|
||||
|
||||
@classmethod
|
||||
def _read_command_output(cls, handle: BinaryIO) -> str:
|
||||
size = handle.seek(0, os.SEEK_END)
|
||||
limit = cls._COMMAND_OUTPUT_MAX_BYTES
|
||||
handle.seek(0)
|
||||
if size <= limit:
|
||||
return handle.read(limit).decode("utf-8", errors="replace")
|
||||
head = handle.read(limit // 2).decode("utf-8", errors="replace")
|
||||
handle.seek(-limit // 2, os.SEEK_END)
|
||||
tail = handle.read(limit // 2).decode("utf-8", errors="replace")
|
||||
return f"{head}\n... [command output truncated; {size} bytes total] ...\n{tail}"
|
||||
|
||||
@classmethod
|
||||
def _terminate_command_tree(cls, process: subprocess.Popen) -> None:
|
||||
"""Kill descendants before reaping the shell, with bounded cleanup."""
|
||||
try:
|
||||
if os.name == "nt":
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(process.pid), "/T", "/F"],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=cls._COMMAND_CLEANUP_SECONDS,
|
||||
check=False,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW,
|
||||
)
|
||||
else:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
logger.warning("Failed to terminate sandbox command process tree (pid=%s)", process.pid, exc_info=True)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
try:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=cls._COMMAND_CLEANUP_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Sandbox command did not exit after termination (pid=%s)", process.pid)
|
||||
|
||||
def list_dir(self, path: str, max_depth=2) -> list[str]:
|
||||
resolved_path = self._resolve_path(path)
|
||||
@@ -369,10 +495,10 @@ class LocalSandbox(Sandbox):
|
||||
result.append(f"{reversed_entry}/" if is_dir and not reversed_entry.endswith("/") else reversed_entry)
|
||||
return result
|
||||
|
||||
def read_file(self, path: str) -> str:
|
||||
def read_file(self, path: str) -> str:
|
||||
resolved_path = self._resolve_path(path)
|
||||
try:
|
||||
with open(resolved_path, encoding="utf-8") as f:
|
||||
with self._open_regular_text(resolved_path) as f:
|
||||
content = f.read()
|
||||
# Only reverse-resolve paths in files that were previously written
|
||||
# by write_file (agent-authored content). User-uploaded files,
|
||||
@@ -383,7 +509,90 @@ class LocalSandbox(Sandbox):
|
||||
return content
|
||||
except OSError as e:
|
||||
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
|
||||
raise type(e)(e.errno, e.strerror, path) from None
|
||||
raise type(e)(e.errno, e.strerror, path) from None
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _open_regular_text(path: str, *, errors: str = "strict", newline: str | None = None):
|
||||
# A skill script can create a FIFO inside its workspace. Opening it
|
||||
# before checking the descriptor type would wait forever for a writer.
|
||||
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0))
|
||||
try:
|
||||
mode = os.fstat(fd).st_mode
|
||||
if stat.S_ISDIR(mode):
|
||||
raise IsADirectoryError(errno.EISDIR, "Path is a directory, not a file", path)
|
||||
if not stat.S_ISREG(mode):
|
||||
raise OSError(errno.EINVAL, "Only regular files can be read", path)
|
||||
with os.fdopen(fd, "r", encoding="utf-8", errors=errors, newline=newline, closefd=False) as handle:
|
||||
yield handle
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def read_file_range(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
start_line: int = 1,
|
||||
end_line: int | None = None,
|
||||
max_chars: int = 32_000,
|
||||
) -> tuple[str, int | None, bool]:
|
||||
"""Stream a bounded range without materializing the whole file."""
|
||||
resolved_path = self._resolve_path(path)
|
||||
lines: list[str] = []
|
||||
chars = 0
|
||||
next_line: int | None = None
|
||||
start_line = max(1, start_line)
|
||||
try:
|
||||
with self._open_regular_text(resolved_path, errors="replace", newline="") as handle:
|
||||
number = 0
|
||||
while True:
|
||||
fragment = handle.readline(4097)
|
||||
if not fragment:
|
||||
break
|
||||
number += 1
|
||||
remaining = max_chars - chars - (1 if lines else 0)
|
||||
capture = number >= start_line and (end_line is None or number <= end_line)
|
||||
if end_line is not None and number > end_line:
|
||||
break
|
||||
if capture and remaining <= 0:
|
||||
next_line = number
|
||||
break
|
||||
parts: list[str] = []
|
||||
captured = 0
|
||||
over_budget = False
|
||||
while True:
|
||||
ended = fragment.endswith(("\n", "\r"))
|
||||
body = fragment.rstrip("\r\n") if ended else fragment
|
||||
if capture and remaining > captured:
|
||||
piece = body[: remaining - captured]
|
||||
parts.append(piece)
|
||||
captured += len(piece)
|
||||
if len(piece) < len(body):
|
||||
over_budget = True
|
||||
elif capture and body:
|
||||
over_budget = True
|
||||
# The next request starts from the file's beginning,
|
||||
# so continuation does not require draining the rest
|
||||
# of an arbitrarily large physical line now.
|
||||
if ended or (capture and captured >= remaining):
|
||||
break
|
||||
fragment = handle.readline(4097)
|
||||
if not fragment:
|
||||
break
|
||||
if not capture:
|
||||
continue
|
||||
line = "".join(parts)
|
||||
lines.append(line)
|
||||
chars += len(line) + (1 if len(lines) > 1 else 0)
|
||||
if over_budget or chars >= max_chars:
|
||||
next_line = number + 1
|
||||
break
|
||||
content = "\n".join(lines)
|
||||
if resolved_path in self._agent_written_paths:
|
||||
content = self._reverse_resolve_paths_in_output(content)
|
||||
return content, next_line, next_line is not None
|
||||
except OSError as e:
|
||||
raise type(e)(e.errno, e.strerror, path) from None
|
||||
|
||||
def write_file(self, path: str, content: str, append: bool = False) -> None:
|
||||
resolved = self._resolve_path_with_mapping(path)
|
||||
@@ -397,9 +606,11 @@ class LocalSandbox(Sandbox):
|
||||
# Resolve container paths in content to local paths
|
||||
# using the content-specific resolver (forward-slash safe)
|
||||
resolved_content = self._resolve_paths_in_content(content)
|
||||
mode = "a" if append else "w"
|
||||
with open(resolved_path, mode, encoding="utf-8") as f:
|
||||
f.write(resolved_content)
|
||||
if append:
|
||||
with open(resolved_path, "a", encoding="utf-8") as f:
|
||||
f.write(resolved_content)
|
||||
else:
|
||||
self._atomic_overwrite(resolved_path, resolved_content)
|
||||
# Track this path so read_file knows to reverse-resolve on read.
|
||||
# Only agent-written files get reverse-resolved; user uploads and
|
||||
# external tool output are left untouched.
|
||||
@@ -408,6 +619,48 @@ class LocalSandbox(Sandbox):
|
||||
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
|
||||
raise type(e)(e.errno, e.strerror, path) from None
|
||||
|
||||
@staticmethod
|
||||
def _atomic_overwrite(path: str, content: str) -> None:
|
||||
"""Publish a complete file only after encoding and disk writes succeed."""
|
||||
# Preserve ordinary open(..., "w") symlink-following semantics.
|
||||
target = os.path.realpath(path)
|
||||
previous_mode: int | None = None
|
||||
try:
|
||||
previous_stat = os.stat(target)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
if stat.S_ISDIR(previous_stat.st_mode):
|
||||
raise IsADirectoryError(errno.EISDIR, "Path is a directory", path)
|
||||
if not stat.S_ISREG(previous_stat.st_mode):
|
||||
raise OSError(errno.EINVAL, "Target is not a regular file", path)
|
||||
# Replacement depends on directory permissions; also enforce the
|
||||
# existing file's write permissions, without truncating it.
|
||||
descriptor = os.open(target, os.O_WRONLY)
|
||||
os.close(descriptor)
|
||||
previous_mode = stat.S_IMODE(previous_stat.st_mode)
|
||||
|
||||
staged_path: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", encoding="utf-8", dir=os.path.dirname(target),
|
||||
prefix=".deerflow-write-", suffix=".tmp", delete=False,
|
||||
) as staged:
|
||||
staged_path = staged.name
|
||||
staged.write(content)
|
||||
staged.flush()
|
||||
os.fsync(staged.fileno())
|
||||
if previous_mode is not None:
|
||||
os.chmod(staged_path, previous_mode)
|
||||
os.replace(staged_path, target)
|
||||
staged_path = None
|
||||
finally:
|
||||
if staged_path is not None:
|
||||
try:
|
||||
os.unlink(staged_path)
|
||||
except OSError:
|
||||
logger.warning("Failed to remove staged sandbox write")
|
||||
|
||||
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
|
||||
resolved_path = Path(self._resolve_path(path))
|
||||
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results)
|
||||
|
||||
+13
-10
@@ -1,9 +1,10 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||||
from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.sandbox_provider import SandboxProvider
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||||
from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.sandbox_provider import SandboxProvider
|
||||
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,13 +30,15 @@ class LocalSandboxProvider(SandboxProvider):
|
||||
"""
|
||||
mappings: list[PathMapping] = []
|
||||
|
||||
# Map skills container path to local skills directory
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
config = get_app_config()
|
||||
skills_path = config.skills.get_skills_path()
|
||||
container_path = config.skills.container_path
|
||||
# Map the model-facing path to the online-only materialized view. The
|
||||
# canonical tree also contains offline skills and is administration
|
||||
# state, not a sandbox capability.
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
config = get_app_config()
|
||||
skills_path = refresh_skill_sandbox_view().root
|
||||
container_path = config.skills.container_path
|
||||
|
||||
# Only add mapping if skills directory exists
|
||||
if skills_path.exists():
|
||||
|
||||
@@ -28,7 +28,7 @@ class Sandbox(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def read_file(self, path: str) -> str:
|
||||
def read_file(self, path: str) -> str:
|
||||
"""Read the content of a file.
|
||||
|
||||
Args:
|
||||
@@ -37,7 +37,38 @@ class Sandbox(ABC):
|
||||
Returns:
|
||||
The content of the file.
|
||||
"""
|
||||
pass
|
||||
pass
|
||||
|
||||
def read_file_range(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
start_line: int = 1,
|
||||
end_line: int | None = None,
|
||||
max_chars: int = 32_000,
|
||||
) -> tuple[str, int | None, bool]:
|
||||
"""Read a bounded line range.
|
||||
|
||||
Providers should override this method to avoid loading the entire file.
|
||||
The compatibility implementation remains bounded at the tool boundary,
|
||||
though an older remote provider may still materialize its response.
|
||||
"""
|
||||
content = self.read_file(path)
|
||||
selected: list[str] = []
|
||||
chars = 0
|
||||
next_line: int | None = None
|
||||
for number, line in enumerate(content.splitlines(), start=1):
|
||||
if number < start_line:
|
||||
continue
|
||||
if end_line is not None and number > end_line:
|
||||
break
|
||||
addition = len(line) + (1 if selected else 0)
|
||||
if chars + addition > max_chars:
|
||||
next_line = number
|
||||
break
|
||||
selected.append(line)
|
||||
chars += addition
|
||||
return "\n".join(selected), next_line, next_line is not None
|
||||
|
||||
@abstractmethod
|
||||
def list_dir(self, path: str, max_depth=2) -> list[str]:
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import fnmatch
|
||||
import os
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
".git",
|
||||
".svn",
|
||||
@@ -56,8 +57,10 @@ IGNORE_PATTERNS = [
|
||||
".ruff_cache",
|
||||
]
|
||||
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
|
||||
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024
|
||||
DEFAULT_MAX_TOTAL_SCAN_BYTES = 64 * 1024 * 1024
|
||||
DEFAULT_MAX_SCAN_SECONDS = 5.0
|
||||
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -148,7 +151,9 @@ def find_grep_matches(
|
||||
literal: bool = False,
|
||||
case_sensitive: bool = False,
|
||||
max_results: int = 100,
|
||||
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
|
||||
max_file_size: int | None = None,
|
||||
max_total_scan_bytes: int = DEFAULT_MAX_TOTAL_SCAN_BYTES,
|
||||
max_scan_seconds: float = DEFAULT_MAX_SCAN_SECONDS,
|
||||
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
|
||||
) -> tuple[list[GrepMatch], bool]:
|
||||
matches: list[GrepMatch] = []
|
||||
@@ -162,16 +167,31 @@ def find_grep_matches(
|
||||
|
||||
regex_source = re.escape(pattern) if literal else pattern
|
||||
flags = 0 if case_sensitive else re.IGNORECASE
|
||||
regex = re.compile(regex_source, flags)
|
||||
|
||||
# Skip lines longer than this to prevent ReDoS on minified / no-newline files.
|
||||
_max_line_chars = line_summary_length * 10
|
||||
regex = re.compile(regex_source, flags)
|
||||
|
||||
if max_file_size is None:
|
||||
try:
|
||||
from deerflow.uploads.policy import get_runtime_upload_policy
|
||||
|
||||
groups = (get_runtime_upload_policy().get("groups") or {}).values()
|
||||
configured = max((int(group.get("max_bytes") or 0) for group in groups), default=DEFAULT_MAX_FILE_SIZE_BYTES)
|
||||
max_file_size = min(200 * 1024 * 1024, max(DEFAULT_MAX_FILE_SIZE_BYTES, configured))
|
||||
except Exception:
|
||||
max_file_size = DEFAULT_MAX_FILE_SIZE_BYTES
|
||||
|
||||
# Read lines in bounded fragments to prevent a minified / no-newline file
|
||||
# from allocating its entire body before the length check can run.
|
||||
_max_line_chars = line_summary_length * 10
|
||||
deadline = time.monotonic() + max(0.1, max_scan_seconds)
|
||||
scanned_bytes = 0
|
||||
|
||||
for current_root, dirs, files in os.walk(root):
|
||||
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
||||
rel_dir = Path(current_root).relative_to(root)
|
||||
|
||||
for name in files:
|
||||
for name in files:
|
||||
if time.monotonic() >= deadline:
|
||||
return matches, True
|
||||
if should_ignore_name(name):
|
||||
continue
|
||||
|
||||
@@ -187,13 +207,32 @@ def find_grep_matches(
|
||||
file_path = candidate_path.resolve()
|
||||
if not file_path.is_relative_to(root):
|
||||
continue
|
||||
if file_path.stat().st_size > max_file_size or is_binary_file(file_path):
|
||||
continue
|
||||
with file_path.open(encoding="utf-8", errors="replace") as handle:
|
||||
for line_number, line in enumerate(handle, start=1):
|
||||
if len(line) > _max_line_chars:
|
||||
continue
|
||||
if regex.search(line):
|
||||
file_size = file_path.stat().st_size
|
||||
if file_size > max_file_size:
|
||||
truncated = True
|
||||
continue
|
||||
if scanned_bytes + file_size > max_total_scan_bytes:
|
||||
return matches, True
|
||||
scanned_bytes += file_size
|
||||
if is_binary_file(file_path):
|
||||
continue
|
||||
with file_path.open(encoding="utf-8", errors="replace") as handle:
|
||||
line_number = 0
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
return matches, True
|
||||
line = handle.readline(_max_line_chars + 1)
|
||||
if not line:
|
||||
break
|
||||
line_number += 1
|
||||
if len(line) > _max_line_chars and not line.endswith(("\n", "\r")):
|
||||
# Drain the rest of this physical line in bounded
|
||||
# fragments, but never run an attacker-controlled
|
||||
# regex over it.
|
||||
while line and not line.endswith(("\n", "\r")):
|
||||
line = handle.readline(_max_line_chars + 1)
|
||||
continue
|
||||
if regex.search(line):
|
||||
matches.append(
|
||||
GrepMatch(
|
||||
path=str(file_path),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+41
@@ -0,0 +1,41 @@
|
||||
"""A skill snapshot is reusable only inside one explicitly enabled tool call."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from functools import wraps
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from deerflow.config.runtime_performance_config import get_runtime_performance_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.sandbox.skill_view import SkillSandboxView
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillViewScope:
|
||||
view: "SkillSandboxView | None" = None
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
_scope: ContextVar[SkillViewScope | None] = ContextVar("sandbox_skill_view_scope", default=None)
|
||||
|
||||
|
||||
def get_active_skill_view_scope() -> SkillViewScope | None:
|
||||
return _scope.get()
|
||||
|
||||
|
||||
def with_skill_view_scope[**P, R](func: Callable[P, R]) -> Callable[P, R]:
|
||||
@wraps(func)
|
||||
def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
if not get_runtime_performance_config().skill_view_reuse_enabled:
|
||||
return func(*args, **kwargs)
|
||||
token = _scope.set(SkillViewScope())
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
# Do not leak a previously enabled skill into the next tool call,
|
||||
# including when a tool raises or shares an executor thread.
|
||||
_scope.reset(token)
|
||||
|
||||
return wrapped
|
||||
@@ -1,14 +1,21 @@
|
||||
import json
|
||||
import posixpath
|
||||
import re
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from inspect import signature
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langgraph.typing import ContextT
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from deerflow.agents.thread_state import ThreadDataState, ThreadState
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.paths import VIRTUAL_PATH_PREFIX
|
||||
from deerflow.html_runtime.html_postprocess import maybe_fix_generated_html
|
||||
from deerflow.sandbox.exceptions import (
|
||||
SandboxError,
|
||||
SandboxNotFoundError,
|
||||
@@ -19,6 +26,7 @@ from deerflow.sandbox.sandbox import Sandbox
|
||||
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
|
||||
from deerflow.sandbox.search import GrepMatch
|
||||
from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed
|
||||
from deerflow.sandbox.skill_view_scope import get_active_skill_view_scope, with_skill_view_scope
|
||||
|
||||
_ABSOLUTE_PATH_PATTERN = re.compile(r"(?<![:\w])(?<!:/)/(?:[^\s\"'`;&|<>()]+)")
|
||||
_FILE_URL_PATTERN = re.compile(r"\bfile://\S+", re.IGNORECASE)
|
||||
@@ -99,36 +107,189 @@ def _get_skills_container_path() -> str:
|
||||
|
||||
|
||||
def _get_skills_host_path() -> str | None:
|
||||
"""Get the skills host filesystem path from config.
|
||||
"""Get the online-only model-facing skills path.
|
||||
|
||||
Returns None if the skills directory does not exist or config cannot be
|
||||
loaded. Only successful lookups are cached; failures are retried on the
|
||||
next call so that a transiently unavailable skills directory does not
|
||||
permanently disable skills access.
|
||||
Never return the canonical administrative skills tree. Offline skills
|
||||
intentionally remain there and would otherwise be reachable from bash by
|
||||
constructing a path dynamically instead of spelling it in the tool call.
|
||||
A warm cache resolves the published view without stating every skill file.
|
||||
"""
|
||||
cached = getattr(_get_skills_host_path, "_cached", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.sandbox.skill_view import current_skill_view_root
|
||||
|
||||
config = get_app_config()
|
||||
skills_path = config.skills.get_skills_path()
|
||||
if skills_path.exists():
|
||||
value = str(skills_path)
|
||||
_get_skills_host_path._cached = value # type: ignore[attr-defined]
|
||||
return value
|
||||
return str(current_skill_view_root())
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _refresh_skill_capability() -> None:
|
||||
"""Refresh the filesystem capability used by local and AIO sandboxes."""
|
||||
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
|
||||
|
||||
refresh_skill_sandbox_view()
|
||||
|
||||
|
||||
def _prepare_skill_paths(paths: list[str]) -> None:
|
||||
"""Refresh only the skills a tool is about to read, when the cache is on."""
|
||||
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
|
||||
|
||||
if not _skill_view_cache_enabled():
|
||||
_refresh_skill_capability()
|
||||
return
|
||||
prepare_skill_tool_view(paths, container_path=_get_skills_container_path())
|
||||
|
||||
|
||||
def _prepare_command_skill_capability(command: str) -> None:
|
||||
"""Skip the skill-view check when a command does not name a skill.
|
||||
|
||||
The online view is still the security boundary. Management writes remove
|
||||
an offlined skill from that view immediately, so a later command that
|
||||
hides the path cannot run it. Commands that spell a skill path reconcile
|
||||
only that skill.
|
||||
"""
|
||||
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
|
||||
|
||||
if not _skill_view_cache_enabled():
|
||||
_refresh_skill_capability()
|
||||
return
|
||||
skills_container = _get_skills_container_path()
|
||||
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
|
||||
if boundary.search(command) is None:
|
||||
from deerflow.sandbox.skill_view import note_skill_view_check
|
||||
|
||||
note_skill_view_check(status="skipped")
|
||||
return
|
||||
paths: list[str] = []
|
||||
url_spans = _non_file_url_spans(command)
|
||||
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
|
||||
if _is_in_spans(match.start(), url_spans):
|
||||
continue
|
||||
path = match.group()
|
||||
if _is_skills_path(path):
|
||||
paths.append(path)
|
||||
prepare_skill_tool_view(paths, container_path=skills_container)
|
||||
|
||||
|
||||
def _skill_paths_in_text(text: str) -> list[str]:
|
||||
paths: list[str] = []
|
||||
for match in _ABSOLUTE_PATH_PATTERN.finditer(text):
|
||||
path = match.group()
|
||||
if _is_skills_path(path) and path not in paths:
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def _finish_skill_view_trace(tool: str, command: str, started: float, *, always: bool) -> None:
|
||||
"""Write the diagnostic row only after this tool call has returned."""
|
||||
try:
|
||||
from deerflow.sandbox.skill_view import commit_skill_view_trace, skill_names_for_paths
|
||||
|
||||
commit_skill_view_trace(
|
||||
tool=tool,
|
||||
command=command,
|
||||
skill_names=skill_names_for_paths(_skill_paths_in_text(command), container_path=_get_skills_container_path()),
|
||||
tool_ms=(time.perf_counter() - started) * 1000,
|
||||
allow_empty=always,
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _trace_completed_tool(tool_name: str, *, command_from: str, always: bool = False) -> Callable:
|
||||
"""Publish one timing row after the wrapped tool returns, including on error."""
|
||||
|
||||
def decorator[**P, R](fn: Callable[P, R]) -> Callable[P, R]:
|
||||
sig = signature(fn)
|
||||
|
||||
@wraps(fn)
|
||||
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
from deerflow.sandbox.skill_view import begin_skill_view_trace, end_skill_view_trace
|
||||
|
||||
bound = sig.bind_partial(*args, **kwargs)
|
||||
command = str(bound.arguments.get(command_from) or "")
|
||||
if tool_name in {"glob", "grep"}:
|
||||
command = f"{bound.arguments.get('pattern') or ''} {command}".strip()
|
||||
started = time.perf_counter()
|
||||
token = begin_skill_view_trace()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
try:
|
||||
_finish_skill_view_trace(tool_name, command, started, always=always)
|
||||
finally:
|
||||
end_skill_view_trace(token)
|
||||
|
||||
wrapper.__signature__ = sig # type: ignore[attr-defined]
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _is_skills_path(path: str) -> bool:
|
||||
"""Check if a path is under the skills container path."""
|
||||
skills_prefix = _get_skills_container_path()
|
||||
return path == skills_prefix or path.startswith(f"{skills_prefix}/")
|
||||
|
||||
|
||||
def _require_enabled_skill_path(path: str) -> None:
|
||||
"""Fail closed unless ``path`` belongs to a currently enabled skill."""
|
||||
scope = get_active_skill_view_scope()
|
||||
if scope is not None:
|
||||
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
|
||||
|
||||
# The entry snapshot already validated availability and discovered the
|
||||
# real skill directories. Re-parsing the full catalogue here (and again
|
||||
# during local path validation) would defeat per-call view reuse.
|
||||
view = refresh_skill_sandbox_view()
|
||||
prefix = _get_skills_container_path().replace("\\", "/").rstrip("/")
|
||||
normalized = path.replace("\\", "/").rstrip("/")
|
||||
if ".." in PurePosixPath(normalized).parts:
|
||||
raise PermissionError("Skill path traversal is not allowed")
|
||||
for _name, category, relative in view.enabled_roots:
|
||||
# Root/category-wide access is never a model capability.
|
||||
if not relative or relative == ".":
|
||||
continue
|
||||
skill_root = f"{prefix}/{category}/{relative}"
|
||||
if normalized == skill_root or normalized.startswith(f"{skill_root}/"):
|
||||
return
|
||||
raise PermissionError("Skill path is unavailable")
|
||||
|
||||
from deerflow.tools.builtins.skill_tools import require_enabled_skill_path
|
||||
|
||||
require_enabled_skill_path(path, container_path=_get_skills_container_path())
|
||||
|
||||
|
||||
def _validate_enabled_skill_paths_in_command(command: str) -> None:
|
||||
"""Give a clear error for explicit unavailable skill paths.
|
||||
|
||||
This is a usability policy, not the security boundary. Shell expansion
|
||||
can hide paths from string inspection, so the enforceable boundary is the
|
||||
online-only filesystem view. Management writes update that view immediately;
|
||||
a command that names a skill reconciles that skill before it runs.
|
||||
"""
|
||||
skills_container = _get_skills_container_path()
|
||||
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
|
||||
if boundary.search(command) is None:
|
||||
return
|
||||
|
||||
matched = False
|
||||
url_spans = _non_file_url_spans(command)
|
||||
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
|
||||
if _is_in_spans(match.start(), url_spans):
|
||||
continue
|
||||
path = match.group()
|
||||
if not _is_skills_path(path):
|
||||
continue
|
||||
matched = True
|
||||
_require_enabled_skill_path(path)
|
||||
|
||||
# Quoting/concatenation that hides the concrete skill directory is not a
|
||||
# valid way around the gate. Exact, discoverable paths are required.
|
||||
if not matched:
|
||||
raise PermissionError("Skill path is unavailable")
|
||||
|
||||
|
||||
def _resolve_skills_path(path: str) -> str:
|
||||
"""Resolve a virtual skills path to a host filesystem path.
|
||||
|
||||
@@ -649,6 +810,7 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *,
|
||||
|
||||
# Skills paths — read-only access only
|
||||
if _is_skills_path(path):
|
||||
_require_enabled_skill_path(path)
|
||||
if not read_only:
|
||||
raise PermissionError(f"Write access to skills path is not allowed: {path}")
|
||||
return
|
||||
@@ -1268,6 +1430,8 @@ def _truncate_ls_output(output: str, max_chars: int) -> str:
|
||||
|
||||
|
||||
@tool("bash", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
@_trace_completed_tool("bash", command_from="command", always=True)
|
||||
def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str:
|
||||
"""Execute a bash command in a Linux environment.
|
||||
|
||||
@@ -1281,6 +1445,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
|
||||
command: The bash command to execute. Always use absolute paths for files and directories.
|
||||
"""
|
||||
try:
|
||||
_prepare_command_skill_capability(command)
|
||||
_validate_enabled_skill_paths_in_command(command)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
if is_local_sandbox(runtime):
|
||||
if not is_host_bash_allowed():
|
||||
@@ -1317,6 +1483,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
|
||||
|
||||
|
||||
@tool("ls", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
@_trace_completed_tool("ls", command_from="path")
|
||||
def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: str) -> str:
|
||||
"""List the contents of a directory up to 2 levels deep in tree format.
|
||||
|
||||
@@ -1324,10 +1492,13 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
|
||||
description: Explain why you are listing this directory in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
|
||||
path: The **absolute** path to the directory to list.
|
||||
"""
|
||||
requested_path = path
|
||||
try:
|
||||
if _is_skills_path(path):
|
||||
_prepare_skill_paths([path])
|
||||
_require_enabled_skill_path(path)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
requested_path = path
|
||||
thread_data = None
|
||||
if is_local_sandbox(runtime):
|
||||
thread_data = get_thread_data(runtime)
|
||||
@@ -1364,6 +1535,8 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
|
||||
|
||||
|
||||
@tool("glob", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
@_trace_completed_tool("glob", command_from="path")
|
||||
def glob_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str,
|
||||
@@ -1381,10 +1554,13 @@ def glob_tool(
|
||||
include_dirs: Whether matching directories should also be returned. Default is False.
|
||||
max_results: Maximum number of paths to return. Default is 200.
|
||||
"""
|
||||
requested_path = path
|
||||
try:
|
||||
if _is_skills_path(path):
|
||||
_prepare_skill_paths([path])
|
||||
_require_enabled_skill_path(path)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
requested_path = path
|
||||
effective_max_results = _resolve_max_results(
|
||||
"glob",
|
||||
max_results,
|
||||
@@ -1414,6 +1590,8 @@ def glob_tool(
|
||||
|
||||
|
||||
@tool("grep", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
@_trace_completed_tool("grep", command_from="path")
|
||||
def grep_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str,
|
||||
@@ -1435,10 +1613,13 @@ def grep_tool(
|
||||
case_sensitive: Whether matching is case-sensitive. Default is False.
|
||||
max_results: Maximum number of matching lines to return. Default is 100.
|
||||
"""
|
||||
requested_path = path
|
||||
try:
|
||||
if _is_skills_path(path):
|
||||
_prepare_skill_paths([path])
|
||||
_require_enabled_skill_path(path)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
requested_path = path
|
||||
effective_max_results = _resolve_max_results(
|
||||
"grep",
|
||||
max_results,
|
||||
@@ -1484,6 +1665,8 @@ def grep_tool(
|
||||
|
||||
|
||||
@tool("read_file", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
@_trace_completed_tool("read_file", command_from="path")
|
||||
def read_file_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str,
|
||||
@@ -1501,15 +1684,27 @@ def read_file_tool(
|
||||
"""
|
||||
try:
|
||||
requested_path = path
|
||||
if start_line is not None and (not isinstance(start_line, int) or isinstance(start_line, bool) or start_line < 1):
|
||||
return "Error: start_line must be a positive integer"
|
||||
if end_line is not None and (not isinstance(end_line, int) or isinstance(end_line, bool) or end_line < 1):
|
||||
return "Error: end_line must be a positive integer"
|
||||
effective_start = start_line or 1
|
||||
if end_line is not None and end_line < effective_start:
|
||||
return "Error: end_line must be greater than or equal to start_line"
|
||||
if _is_skills_path(requested_path):
|
||||
_prepare_skill_paths([requested_path])
|
||||
from deerflow.tools.builtins.skill_tools import (
|
||||
_resolve_active_agent_id,
|
||||
can_access_coordinator_only_skill,
|
||||
is_coordinator_only_skill_path,
|
||||
)
|
||||
|
||||
if is_coordinator_only_skill_path(requested_path) and _resolve_active_agent_id(runtime) != "roundtable-coordinator":
|
||||
if is_coordinator_only_skill_path(requested_path) and not can_access_coordinator_only_skill(
|
||||
_resolve_active_agent_id(runtime), "agent_orchestration"
|
||||
):
|
||||
# Match skill_view's non-leaking behavior: ordinary agents must not discover whether it exists.
|
||||
return f"Error: File not found: {requested_path}"
|
||||
_require_enabled_skill_path(requested_path)
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
if is_local_sandbox(runtime):
|
||||
@@ -1522,21 +1717,62 @@ def read_file_tool(
|
||||
elif not _is_custom_mount_path(path):
|
||||
path = _resolve_and_validate_user_data_path(path, thread_data)
|
||||
# Custom mount paths are resolved by LocalSandbox._resolve_path()
|
||||
content = sandbox.read_file(path)
|
||||
try:
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.uploads.policy import get_runtime_large_file_policy
|
||||
|
||||
sandbox_cfg = get_app_config().sandbox
|
||||
configured_max = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
|
||||
large_file_policy = get_runtime_large_file_policy()
|
||||
except Exception:
|
||||
configured_max = 50000
|
||||
large_file_policy = {
|
||||
"threshold_bytes": 1024 * 1024,
|
||||
"max_result_chars": 12000,
|
||||
"hard_max_result_chars": 32000,
|
||||
}
|
||||
policy_max = max(256, int(large_file_policy.get("max_result_chars", 12000)))
|
||||
hard_max = max(256, min(32000, int(large_file_policy.get("hard_max_result_chars", 32000))))
|
||||
max_chars = min(policy_max, hard_max, configured_max if configured_max > 0 else hard_max)
|
||||
|
||||
# A range-free read of a large local file returns deterministic
|
||||
# head/middle/tail samples plus whole-file streaming statistics. The
|
||||
# raw body is never materialized in memory or sent to the model.
|
||||
if start_line is None and end_line is None and is_local_sandbox(runtime):
|
||||
try:
|
||||
physical_path = Path(path)
|
||||
threshold = max(1, int(large_file_policy.get("threshold_bytes", 1024 * 1024)))
|
||||
if physical_path.is_file() and physical_path.stat().st_size >= threshold:
|
||||
from deerflow.uploads.inspection import inspect_text_file
|
||||
|
||||
inspected = inspect_text_file(
|
||||
physical_path,
|
||||
display_path=requested_path,
|
||||
max_result_chars=max_chars,
|
||||
hard_max_result_chars=hard_max,
|
||||
)
|
||||
_record_skill_read(requested_path)
|
||||
return inspected["rendered"]
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Reserve room for a continuation marker so the complete tool result,
|
||||
# not just its body, remains under the hard cap.
|
||||
body_budget = max(64, max_chars - 180)
|
||||
content, next_line, truncated = sandbox.read_file_range(
|
||||
path,
|
||||
start_line=effective_start,
|
||||
end_line=end_line,
|
||||
max_chars=body_budget,
|
||||
)
|
||||
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
|
||||
# 非技能路径会被 _record_skill_read 自行忽略。
|
||||
_record_skill_read(requested_path)
|
||||
if not content:
|
||||
if not content and not truncated:
|
||||
return "(empty)"
|
||||
if start_line is not None and end_line is not None:
|
||||
content = "\n".join(content.splitlines()[start_line - 1 : end_line])
|
||||
try:
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
sandbox_cfg = get_app_config().sandbox
|
||||
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
|
||||
except Exception:
|
||||
max_chars = 50000
|
||||
if truncated:
|
||||
continuation = next_line or effective_start
|
||||
content = f"{content}\n... [bounded read; continue with start_line={continuation}]"
|
||||
return _truncate_read_file_output(content, max_chars)
|
||||
except SandboxError as e:
|
||||
return f"Error: {e}"
|
||||
@@ -1551,11 +1787,12 @@ def read_file_tool(
|
||||
|
||||
|
||||
@tool("write_file", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
def write_file_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str,
|
||||
path: str,
|
||||
content: str,
|
||||
description: str = "",
|
||||
path: str = "",
|
||||
content: str = "",
|
||||
append: bool = False,
|
||||
) -> str:
|
||||
"""Write text content to a file.
|
||||
@@ -1565,6 +1802,13 @@ def write_file_tool(
|
||||
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
|
||||
content: The content to write to the file. ALWAYS PROVIDE THIS PARAMETER THIRD.
|
||||
"""
|
||||
if not str(path).strip():
|
||||
return "Error: write_file requires an absolute path."
|
||||
if content is None or content == "":
|
||||
return (
|
||||
"Error: write_file requires the full file content in the same call as path. "
|
||||
"Retry once with path and content together."
|
||||
)
|
||||
try:
|
||||
sandbox = ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
@@ -1576,6 +1820,7 @@ def write_file_tool(
|
||||
path = _resolve_and_validate_user_data_path(path, thread_data)
|
||||
# Custom mount paths are resolved by LocalSandbox._resolve_path()
|
||||
with get_file_operation_lock(sandbox, path):
|
||||
content = maybe_fix_generated_html(path, content)
|
||||
sandbox.write_file(path, content, append)
|
||||
return "OK"
|
||||
except SandboxError as e:
|
||||
@@ -1590,7 +1835,90 @@ def write_file_tool(
|
||||
return f"Error: Unexpected error writing file: {_sanitize_error(e, runtime)}"
|
||||
|
||||
|
||||
class _BeginWriteFileInput(BaseModel):
|
||||
"""Small, strict prepare envelope; file bytes are never valid here."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
||||
|
||||
runtime: ToolRuntime[ContextT, ThreadState]
|
||||
description: str = Field(
|
||||
default="",
|
||||
max_length=240,
|
||||
description="A short explanation of the file being written.",
|
||||
)
|
||||
path: str = Field(
|
||||
min_length=1,
|
||||
max_length=4096,
|
||||
description="The absolute sandbox path of the target file.",
|
||||
)
|
||||
append: bool = Field(
|
||||
default=False,
|
||||
description="Whether the final write appends instead of overwriting.",
|
||||
)
|
||||
|
||||
|
||||
@tool(
|
||||
"begin_write_file",
|
||||
args_schema=_BeginWriteFileInput,
|
||||
parse_docstring=True,
|
||||
)
|
||||
@with_skill_view_scope
|
||||
def begin_write_file_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str = "",
|
||||
path: str = "",
|
||||
append: bool = False,
|
||||
) -> str:
|
||||
"""Prepare a file target for a streamed text-body write without writing it yet.
|
||||
|
||||
This internal compatibility handshake is exposed only to models configured
|
||||
with ``write_file_transport: artifact_renderer`` (or the legacy
|
||||
``text_bridge`` mode). It validates the target but writes no bytes. After a
|
||||
``status=ready`` result, DeerFlow obtains the body through the configured
|
||||
transport and executes the real ``write_file`` tool only after validation.
|
||||
|
||||
Args:
|
||||
description: Explain why you are writing to this file in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
|
||||
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
|
||||
append: Whether the final write should append instead of overwrite.
|
||||
"""
|
||||
if not str(path).strip().startswith("/"):
|
||||
return "Error: begin_write_file requires an absolute path."
|
||||
requested_path = path
|
||||
try:
|
||||
# Match write_file's sandbox acquisition and local path authorization,
|
||||
# but deliberately do not take a file lock or persist any bytes.
|
||||
ensure_sandbox_initialized(runtime)
|
||||
ensure_thread_directories_exist(runtime)
|
||||
if is_local_sandbox(runtime):
|
||||
thread_data = get_thread_data(runtime)
|
||||
validate_local_tool_path(path, thread_data)
|
||||
if not _is_custom_mount_path(path):
|
||||
_resolve_and_validate_user_data_path(path, thread_data)
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "ready",
|
||||
"path": requested_path,
|
||||
"description": str(description or ""),
|
||||
"append": append,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
except SandboxError as e:
|
||||
return f"Error: {e}"
|
||||
except PermissionError:
|
||||
return f"Error: Permission denied writing to file: {requested_path}"
|
||||
except IsADirectoryError:
|
||||
return f"Error: Path is a directory, not a file: {requested_path}"
|
||||
except OSError as e:
|
||||
return f"Error: Failed to prepare file '{requested_path}': {_sanitize_error(e, runtime)}"
|
||||
except Exception as e:
|
||||
return f"Error: Unexpected error preparing file: {_sanitize_error(e, runtime)}"
|
||||
|
||||
|
||||
@tool("str_replace", parse_docstring=True)
|
||||
@with_skill_view_scope
|
||||
def str_replace_tool(
|
||||
runtime: ToolRuntime[ContextT, ThreadState],
|
||||
description: str,
|
||||
@@ -1629,6 +1957,7 @@ def str_replace_tool(
|
||||
content = content.replace(old_str, new_str)
|
||||
else:
|
||||
content = content.replace(old_str, new_str, 1)
|
||||
content = maybe_fix_generated_html(path, content)
|
||||
sandbox.write_file(path, content)
|
||||
return "OK"
|
||||
except SandboxError as e:
|
||||
|
||||
Reference in new issue
Block a user