sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -26,7 +26,33 @@ class ResolvedPath(NamedTuple):
mapping: PathMapping | None
class LocalSandbox(Sandbox):
class LocalSandbox(Sandbox):
_MODEL_COMMAND_HIDDEN_ENV_KEYS = frozenset(
{
"DEER_FLOW_CONFIG_PATH",
"DEER_FLOW_EXTENSIONS_CONFIG_PATH",
"DEER_FLOW_HOME",
"DEER_FLOW_HOST_BASE_DIR",
"DEER_FLOW_HOST_SKILLS_PATH",
"DEER_FLOW_PROJECT_ROOT",
"DEER_FLOW_SKILLS_PATH",
"PYTHONPATH",
}
)
@staticmethod
def _model_command_environment() -> dict[str, str]:
"""Return a reduced environment for model-originated host commands.
LocalSandbox is deliberately not an OS security sandbox, but it should
not hand a model canonical skills/config paths through process
environment variables. ``PYTHONPATH`` is also removed so an editable
backend checkout is not advertised as an import root. Non-path
``DEER_FLOW_*`` settings (for example the Gateway port), ordinary PATH,
proxy and skill-specific credentials remain available.
"""
return {key: value for key, value in os.environ.items() if key.upper() not in LocalSandbox._MODEL_COMMAND_HIDDEN_ENV_KEYS}
@staticmethod
def _shell_name(shell: str) -> str:
"""Return the executable name for a shell path or command."""
@@ -37,26 +63,26 @@ class LocalSandbox(Sandbox):
"""Return whether the selected shell is a PowerShell executable."""
return LocalSandbox._shell_name(shell) in {"powershell", "powershell.exe", "pwsh", "pwsh.exe"}
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
@staticmethod
def _is_cmd_shell(shell: str) -> bool:
"""Return whether the selected shell is cmd.exe."""
return LocalSandbox._shell_name(shell) in {"cmd", "cmd.exe"}
@staticmethod
def _normalize_powershell_command(command: str) -> str:
"""Make bash-style command glue usable in Windows PowerShell 5.
The public tool is still named ``bash`` for model compatibility, but on
Windows local sandbox it may fall back to PowerShell. PowerShell 5 does
not support ``&&``, while the sandbox layer prepends commands with
``cd <workspace> &&``. Rewriting the separator keeps skill scripts
executable in that fallback shell.
"""
return command.replace("&&", ";")
@staticmethod
def _find_first_available_shell(candidates: tuple[str, ...]) -> str | None:
"""Return the first executable shell path or command found from candidates."""
for shell in candidates:
if os.path.isabs(shell):
if os.path.isfile(shell) and os.access(shell, os.X_OK):
@@ -246,7 +272,14 @@ class LocalSandbox(Sandbox):
def replace_match(match: re.Match) -> str:
matched_path = match.group(0)
return self._resolve_path(matched_path)
resolved = self._resolve_path(matched_path)
# Normalize to forward slashes: on Windows hosts the resolved path
# contains backslashes that the shell (bash -c) would treat as
# escape characters, corrupting the rewritten command (e.g.
# F:\repo\... losing its separators). Same normalization as
# ``_resolve_paths_in_content``; bash and Windows tools all accept
# forward-slash paths.
return resolved.replace("\\", "/")
return pattern.sub(replace_match, command)
@@ -314,14 +347,14 @@ class LocalSandbox(Sandbox):
resolved_command = self._resolve_paths_in_command(command)
shell = self._get_shell()
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
if os.name == "nt":
if self._is_powershell(shell):
args = [
shell,
"-NoProfile",
"-Command",
self._normalize_powershell_command(resolved_command),
]
elif self._is_cmd_shell(shell):
args = [shell, "/c", resolved_command]
else:
@@ -335,6 +368,7 @@ class LocalSandbox(Sandbox):
encoding="utf-8",
errors="replace",
timeout=600,
env=self._model_command_environment(),
)
else:
args = [shell, "-c", resolved_command]
@@ -346,8 +380,9 @@ class LocalSandbox(Sandbox):
encoding="utf-8",
errors="replace",
timeout=600,
env=self._model_command_environment(),
)
output = result.stdout
output = result.stdout
if result.stderr:
output += f"\nStd Error:\n{result.stderr}" if output else result.stderr
if result.returncode != 0:
@@ -369,7 +404,7 @@ class LocalSandbox(Sandbox):
result.append(f"{reversed_entry}/" if is_dir and not reversed_entry.endswith("/") else reversed_entry)
return result
def read_file(self, path: str) -> str:
def read_file(self, path: str) -> str:
resolved_path = self._resolve_path(path)
try:
with open(resolved_path, encoding="utf-8") as f:
@@ -383,7 +418,70 @@ class LocalSandbox(Sandbox):
return content
except OSError as e:
# Re-raise with the original path for clearer error messages, hiding internal resolved paths
raise type(e)(e.errno, e.strerror, path) from None
raise type(e)(e.errno, e.strerror, path) from None
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Stream a bounded range without materializing the whole file."""
resolved_path = self._resolve_path(path)
lines: list[str] = []
chars = 0
next_line: int | None = None
start_line = max(1, start_line)
try:
with open(resolved_path, encoding="utf-8", errors="replace", newline="") as handle:
number = 0
while True:
fragment = handle.readline(4097)
if not fragment:
break
number += 1
remaining = max_chars - chars - (1 if lines else 0)
capture = number >= start_line and (end_line is None or number <= end_line)
if end_line is not None and number > end_line:
break
if capture and remaining <= 0:
next_line = number
break
parts: list[str] = []
captured = 0
over_budget = False
while True:
ended = fragment.endswith(("\n", "\r"))
body = fragment.rstrip("\r\n") if ended else fragment
if capture and remaining > captured:
piece = body[: remaining - captured]
parts.append(piece)
captured += len(piece)
if len(piece) < len(body):
over_budget = True
elif capture and body:
over_budget = True
if ended:
break
fragment = handle.readline(4097)
if not fragment:
break
if not capture:
continue
line = "".join(parts)
lines.append(line)
chars += len(line) + (1 if len(lines) > 1 else 0)
if over_budget or chars >= max_chars:
next_line = number + 1
break
content = "\n".join(lines)
if resolved_path in self._agent_written_paths:
content = self._reverse_resolve_paths_in_output(content)
return content, next_line, next_line is not None
except OSError as e:
raise type(e)(e.errno, e.strerror, path) from None
def write_file(self, path: str, content: str, append: bool = False) -> None:
resolved = self._resolve_path_with_mapping(path)
@@ -1,9 +1,10 @@
import logging
from pathlib import Path
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import SandboxProvider
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import SandboxProvider
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
logger = logging.getLogger(__name__)
@@ -29,13 +30,15 @@ class LocalSandboxProvider(SandboxProvider):
"""
mappings: list[PathMapping] = []
# Map skills container path to local skills directory
try:
from deerflow.config import get_app_config
config = get_app_config()
skills_path = config.skills.get_skills_path()
container_path = config.skills.container_path
# Map the model-facing path to the online-only materialized view. The
# canonical tree also contains offline skills and is administration
# state, not a sandbox capability.
try:
from deerflow.config import get_app_config
config = get_app_config()
skills_path = refresh_skill_sandbox_view().root
container_path = config.skills.container_path
# Only add mapping if skills directory exists
if skills_path.exists():
@@ -28,7 +28,7 @@ class Sandbox(ABC):
pass
@abstractmethod
def read_file(self, path: str) -> str:
def read_file(self, path: str) -> str:
"""Read the content of a file.
Args:
@@ -37,7 +37,38 @@ class Sandbox(ABC):
Returns:
The content of the file.
"""
pass
pass
def read_file_range(
self,
path: str,
*,
start_line: int = 1,
end_line: int | None = None,
max_chars: int = 32_000,
) -> tuple[str, int | None, bool]:
"""Read a bounded line range.
Providers should override this method to avoid loading the entire file.
The compatibility implementation remains bounded at the tool boundary,
though an older remote provider may still materialize its response.
"""
content = self.read_file(path)
selected: list[str] = []
chars = 0
next_line: int | None = None
for number, line in enumerate(content.splitlines(), start=1):
if number < start_line:
continue
if end_line is not None and number > end_line:
break
addition = len(line) + (1 if selected else 0)
if chars + addition > max_chars:
next_line = number
break
selected.append(line)
chars += addition
return "\n".join(selected), next_line, next_line is not None
@abstractmethod
def list_dir(self, path: str, max_depth=2) -> list[str]:
@@ -1,9 +1,10 @@
import fnmatch
import os
import re
import os
import re
import time
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
IGNORE_PATTERNS = [
".git",
".svn",
@@ -56,8 +57,10 @@ IGNORE_PATTERNS = [
".ruff_cache",
]
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
DEFAULT_LINE_SUMMARY_LENGTH = 200
DEFAULT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024
DEFAULT_MAX_TOTAL_SCAN_BYTES = 64 * 1024 * 1024
DEFAULT_MAX_SCAN_SECONDS = 5.0
DEFAULT_LINE_SUMMARY_LENGTH = 200
@dataclass(frozen=True)
@@ -148,7 +151,9 @@ def find_grep_matches(
literal: bool = False,
case_sensitive: bool = False,
max_results: int = 100,
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
max_file_size: int | None = None,
max_total_scan_bytes: int = DEFAULT_MAX_TOTAL_SCAN_BYTES,
max_scan_seconds: float = DEFAULT_MAX_SCAN_SECONDS,
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
) -> tuple[list[GrepMatch], bool]:
matches: list[GrepMatch] = []
@@ -162,16 +167,31 @@ def find_grep_matches(
regex_source = re.escape(pattern) if literal else pattern
flags = 0 if case_sensitive else re.IGNORECASE
regex = re.compile(regex_source, flags)
# Skip lines longer than this to prevent ReDoS on minified / no-newline files.
_max_line_chars = line_summary_length * 10
regex = re.compile(regex_source, flags)
if max_file_size is None:
try:
from deerflow.uploads.policy import get_runtime_upload_policy
groups = (get_runtime_upload_policy().get("groups") or {}).values()
configured = max((int(group.get("max_bytes") or 0) for group in groups), default=DEFAULT_MAX_FILE_SIZE_BYTES)
max_file_size = min(200 * 1024 * 1024, max(DEFAULT_MAX_FILE_SIZE_BYTES, configured))
except Exception:
max_file_size = DEFAULT_MAX_FILE_SIZE_BYTES
# Read lines in bounded fragments to prevent a minified / no-newline file
# from allocating its entire body before the length check can run.
_max_line_chars = line_summary_length * 10
deadline = time.monotonic() + max(0.1, max_scan_seconds)
scanned_bytes = 0
for current_root, dirs, files in os.walk(root):
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
rel_dir = Path(current_root).relative_to(root)
for name in files:
for name in files:
if time.monotonic() >= deadline:
return matches, True
if should_ignore_name(name):
continue
@@ -187,13 +207,32 @@ def find_grep_matches(
file_path = candidate_path.resolve()
if not file_path.is_relative_to(root):
continue
if file_path.stat().st_size > max_file_size or is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
for line_number, line in enumerate(handle, start=1):
if len(line) > _max_line_chars:
continue
if regex.search(line):
file_size = file_path.stat().st_size
if file_size > max_file_size:
truncated = True
continue
if scanned_bytes + file_size > max_total_scan_bytes:
return matches, True
scanned_bytes += file_size
if is_binary_file(file_path):
continue
with file_path.open(encoding="utf-8", errors="replace") as handle:
line_number = 0
while True:
if time.monotonic() >= deadline:
return matches, True
line = handle.readline(_max_line_chars + 1)
if not line:
break
line_number += 1
if len(line) > _max_line_chars and not line.endswith(("\n", "\r")):
# Drain the rest of this physical line in bounded
# fragments, but never run an attacker-controlled
# regex over it.
while line and not line.endswith(("\n", "\r")):
line = handle.readline(_max_line_chars + 1)
continue
if regex.search(line):
matches.append(
GrepMatch(
path=str(file_path),
@@ -0,0 +1,346 @@
"""Materialize the model-facing, online-only skills filesystem view.
The canonical skills tree is administration state: it intentionally keeps an
offline skill on disk so that it can be brought online again. It therefore
must not also be the filesystem capability handed to a model sandbox. This
module builds a separate, read-only-at-the-mount-boundary view containing only
currently enabled skills.
The view lives in the writable DeerFlow runtime directory, not below the
canonical root (which production commonly mounts read-only). Files are copied,
not linked: LocalSandbox executes as the backend user, so a model modifying the
disposable view must never mutate canonical skill files through a hard link.
"""
from __future__ import annotations
import json
import logging
import os
import shutil
import tempfile
import threading
import uuid
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from deerflow.skills.storage import SkillStorage
logger = logging.getLogger(__name__)
SKILL_VIEW_DIR_NAME = ".sandbox-online-skills"
SKILL_VIEW_MARKER = ".deerflow-skill-view.json"
SKILL_VIEW_PROTOCOL = "deerflow-online-skills-v1"
SKILL_SUPPORT_MANIFEST = ".deerflow-skill-support.json"
_TOKEN_FILE_NAME = ".sandbox-online-skills.token"
_LOCK_FILE_NAME = ".sandbox-online-skills.lock"
_PROCESS_LOCK = threading.Lock()
_BUILTIN_SUPPORT_DEPENDENCIES: dict[tuple[str, str], frozenset[str]] = {
("public", "task-report-lib"): frozenset(
{
"task-report-import",
"task-report-build",
"task-report-enemy",
"task-report-our",
"task-report-env",
"task-report-judge",
}
),
}
@dataclass(frozen=True)
class SkillSandboxView:
"""A refreshed online-only filesystem capability."""
root: Path
token: str
enabled_names: frozenset[str]
@property
def marker_content(self) -> str:
return json.dumps(
{
"protocol": SKILL_VIEW_PROTOCOL,
"token": self.token,
},
sort_keys=True,
)
def _lock_file(lock_file) -> None:
if os.name == "nt":
import msvcrt
lock_file.seek(0, os.SEEK_END)
if lock_file.tell() == 0:
lock_file.write(b"\0")
lock_file.flush()
lock_file.seek(0)
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
return
import fcntl
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
def _unlock_file(lock_file) -> None:
if os.name == "nt":
import msvcrt
lock_file.seek(0)
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
return
import fcntl
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
@contextmanager
def _view_lock(runtime_root: Path) -> Iterator[None]:
"""Serialize view refreshes in this process and across gateway workers."""
lock_path = runtime_root / _LOCK_FILE_NAME
lock_path.parent.mkdir(parents=True, exist_ok=True)
with _PROCESS_LOCK, lock_path.open("a+b") as lock_file:
_lock_file(lock_file)
try:
yield
finally:
_unlock_file(lock_file)
def _load_or_create_token(runtime_root: Path) -> str:
token_path = runtime_root / _TOKEN_FILE_NAME
try:
token = token_path.read_text(encoding="ascii").strip()
uuid.UUID(token)
return token
except (FileNotFoundError, OSError, UnicodeError, ValueError):
token = str(uuid.uuid4())
token_path.write_text(token + "\n", encoding="ascii")
return token
def _support_roots(skills_root: Path, enabled_names: frozenset[str]) -> list[Path]:
"""Return explicitly declared internal libraries needed by online skills.
Cross-skill code is exceptional: ordinary skills are self-contained. A
directory without ``SKILL.md`` is included only when it carries a manifest
whose ``required_by`` list intersects the enabled skill set. This prevents
an arbitrary unregistered directory from becoming model-readable merely by
being placed next to a skill.
"""
roots: list[Path] = []
for category in ("public", "custom"):
category_root = skills_root / category
if not category_root.is_dir():
continue
for candidate in category_root.iterdir():
if not candidate.is_dir() or candidate.is_symlink():
continue
manifest_path = candidate / SKILL_SUPPORT_MANIFEST
dependencies = set(_BUILTIN_SUPPORT_DEPENDENCIES.get((category, candidate.name), ()))
if manifest_path.is_file():
try:
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
required_by = payload.get("required_by", []) if isinstance(payload, dict) else []
if not isinstance(required_by, list):
raise ValueError("required_by must be a list")
dependencies.update(str(name).strip() for name in required_by if str(name).strip())
except (OSError, UnicodeError, ValueError) as exc:
logger.warning("Ignoring invalid skill support manifest %s: %s", manifest_path, exc)
if dependencies & enabled_names:
roots.append(candidate)
return roots
def _add_tree(
source_root: Path,
skills_root: Path,
desired_files: dict[Path, Path],
desired_dirs: set[Path],
) -> None:
"""Add regular files below ``source_root`` to the desired view snapshot."""
resolved_skills_root = skills_root.resolve()
resolved_source = source_root.resolve()
try:
source_relative = resolved_source.relative_to(resolved_skills_root)
except ValueError as exc:
raise RuntimeError(f"Skill source escapes configured root: {source_root}") from exc
desired_dirs.add(source_relative)
desired_dirs.update(parent for parent in source_relative.parents if parent != Path("."))
for current_root, dir_names, file_names in os.walk(resolved_source, followlinks=False):
current = Path(current_root)
relative_dir = current.relative_to(resolved_skills_root)
desired_dirs.add(relative_dir)
# Symlinks are not copied or followed. Following one could turn a
# harmless online skill into a path to an offline/out-of-tree file.
retained_dirs: list[str] = []
for name in sorted(dir_names):
child = current / name
if child.is_symlink():
logger.warning("Skipping symlink in model-facing skill view: %s", child)
continue
retained_dirs.append(name)
desired_dirs.add(relative_dir / name)
dir_names[:] = retained_dirs
for name in sorted(file_names):
source = current / name
if source.is_symlink() or not source.is_file():
logger.warning("Skipping non-regular file in model-facing skill view: %s", source)
continue
desired_files[relative_dir / name] = source
def _same_file_snapshot(source: Path, destination: Path) -> bool:
try:
source_stat = source.stat()
destination_stat = destination.stat()
except OSError:
return False
return (
source_stat.st_size == destination_stat.st_size
and source_stat.st_mtime_ns == destination_stat.st_mtime_ns
)
def _copy_file_atomic(source: Path, destination: Path) -> None:
destination.parent.mkdir(parents=True, exist_ok=True)
temp_path: Path | None = None
try:
with tempfile.NamedTemporaryFile(delete=False, dir=str(destination.parent)) as temp_file:
temp_path = Path(temp_file.name)
shutil.copy2(source, temp_path)
os.replace(temp_path, destination)
temp_path = None
finally:
if temp_path is not None:
temp_path.unlink(missing_ok=True)
def _remove_stale_entries(
view_root: Path,
desired_files: set[Path],
desired_dirs: set[Path],
) -> None:
marker = Path(SKILL_VIEW_MARKER)
entries = sorted(
view_root.rglob("*"),
key=lambda path: len(path.relative_to(view_root).parts),
reverse=True,
)
for entry in entries:
relative = entry.relative_to(view_root)
if relative == marker:
continue
if entry.is_symlink():
entry.unlink(missing_ok=True)
continue
if entry.is_dir():
if relative not in desired_dirs:
shutil.rmtree(entry)
continue
if relative not in desired_files:
entry.unlink(missing_ok=True)
def _clear_view(view_root: Path) -> None:
if not view_root.exists():
return
for entry in list(view_root.iterdir()):
if entry.is_dir() and not entry.is_symlink():
shutil.rmtree(entry)
else:
entry.unlink(missing_ok=True)
def refresh_skill_sandbox_view(*, storage: SkillStorage | None = None) -> SkillSandboxView:
"""Refresh and return the online-only skills view.
Availability is loaded fresh through ``SkillStorage.load_skills`` on every
call. Any failure clears the capability marker and fails closed; callers
must not fall back to mounting the canonical skills directory.
"""
if storage is None:
from deerflow.skills.storage import get_or_new_skill_storage
storage = get_or_new_skill_storage()
from deerflow.config.paths import get_paths
skills_root = Path(storage.get_skills_root_path()).resolve()
runtime_root = get_paths().base_dir
runtime_root.mkdir(parents=True, exist_ok=True)
view_root = runtime_root / SKILL_VIEW_DIR_NAME
with _view_lock(runtime_root):
view_root.mkdir(parents=True, exist_ok=True)
marker_path = view_root / SKILL_VIEW_MARKER
try:
# Ordinary catalogue discovery deliberately degrades open when the
# extensions document is unreadable. A model capability must fail
# closed, so validate the switch document explicitly and derive
# the enabled subset ourselves.
from deerflow.config.extensions_config import ExtensionsConfig
extensions = ExtensionsConfig.from_file()
discovered_skills = storage.load_skills(enabled_only=False)
enabled_skills = [
skill
for skill in discovered_skills
if extensions.is_skill_enabled(skill.name, skill.category)
]
enabled_names = frozenset(skill.name for skill in enabled_skills)
token = _load_or_create_token(runtime_root)
desired_files: dict[Path, Path] = {}
desired_dirs: set[Path] = {Path("public"), Path("custom")}
for skill in enabled_skills:
_add_tree(Path(skill.skill_dir), skills_root, desired_files, desired_dirs)
for support_root in _support_roots(skills_root, enabled_names):
_add_tree(support_root, skills_root, desired_files, desired_dirs)
for relative_dir in sorted(desired_dirs, key=lambda path: len(path.parts)):
(view_root / relative_dir).mkdir(parents=True, exist_ok=True)
for relative, source in desired_files.items():
destination = view_root / relative
if not _same_file_snapshot(source, destination):
_copy_file_atomic(source, destination)
_remove_stale_entries(view_root, set(desired_files), desired_dirs)
view = SkillSandboxView(root=view_root, token=token, enabled_names=enabled_names)
marker_path.write_text(view.marker_content, encoding="utf-8")
return view
except Exception:
# Without a marker an AIO sandbox cannot attest that it mounted the
# filtered capability. Clear stale content as an additional guard
# for LocalSandbox and then propagate the failure.
_clear_view(view_root)
raise
def expected_skill_view_marker(*, storage: SkillStorage | None = None) -> tuple[Path, str]:
"""Refresh the view and return its marker path plus exact expected text."""
view = refresh_skill_sandbox_view(storage=storage)
return view.root / SKILL_VIEW_MARKER, view.marker_content
__all__ = [
"SKILL_SUPPORT_MANIFEST",
"SKILL_VIEW_DIR_NAME",
"SKILL_VIEW_MARKER",
"SKILL_VIEW_PROTOCOL",
"SkillSandboxView",
"expected_skill_view_marker",
"refresh_skill_sandbox_view",
]
@@ -98,35 +98,69 @@ def _get_skills_container_path() -> str:
return _DEFAULT_SKILLS_CONTAINER_PATH
def _get_skills_host_path() -> str | None:
"""Get the skills host filesystem path from config.
Returns None if the skills directory does not exist or config cannot be
loaded. Only successful lookups are cached; failures are retried on the
next call so that a transiently unavailable skills directory does not
permanently disable skills access.
"""
cached = getattr(_get_skills_host_path, "_cached", None)
if cached is not None:
return cached
try:
from deerflow.config import get_app_config
config = get_app_config()
skills_path = config.skills.get_skills_path()
if skills_path.exists():
value = str(skills_path)
_get_skills_host_path._cached = value # type: ignore[attr-defined]
return value
except Exception:
pass
return None
def _get_skills_host_path() -> str | None:
"""Get the refreshed, online-only model-facing skills path.
Never return the canonical administrative skills tree. Offline skills
intentionally remain there and would otherwise be reachable from bash by
constructing a path dynamically instead of spelling it in the tool call.
"""
try:
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
return str(refresh_skill_sandbox_view().root)
except Exception:
pass
return None
def _refresh_skill_capability() -> None:
"""Refresh the filesystem capability used by local and AIO sandboxes."""
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
refresh_skill_sandbox_view()
def _is_skills_path(path: str) -> bool:
"""Check if a path is under the skills container path."""
skills_prefix = _get_skills_container_path()
return path == skills_prefix or path.startswith(f"{skills_prefix}/")
def _require_enabled_skill_path(path: str) -> None:
"""Fail closed unless ``path`` belongs to a currently enabled skill."""
from deerflow.tools.builtins.skill_tools import require_enabled_skill_path
require_enabled_skill_path(path, container_path=_get_skills_container_path())
def _validate_enabled_skill_paths_in_command(command: str) -> None:
"""Give a clear error for explicit unavailable skill paths.
This is a usability policy, not the security boundary. Shell expansion
can hide paths from string inspection, so the enforceable boundary is the
online-only filesystem view refreshed before every bash invocation.
"""
skills_container = _get_skills_container_path()
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
if boundary.search(command) is None:
return
matched = False
url_spans = _non_file_url_spans(command)
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
if _is_in_spans(match.start(), url_spans):
continue
path = match.group()
if not _is_skills_path(path):
continue
matched = True
_require_enabled_skill_path(path)
# Quoting/concatenation that hides the concrete skill directory is not a
# valid way around the gate. Exact, discoverable paths are required.
if not matched:
raise PermissionError("Skill path is unavailable")
def _resolve_skills_path(path: str) -> str:
@@ -649,6 +683,7 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *,
# Skills paths — read-only access only
if _is_skills_path(path):
_require_enabled_skill_path(path)
if not read_only:
raise PermissionError(f"Write access to skills path is not allowed: {path}")
return
@@ -1268,7 +1303,7 @@ def _truncate_ls_output(output: str, max_chars: int) -> str:
@tool("bash", parse_docstring=True)
def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str:
def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str:
"""Execute a bash command in a Linux environment.
@@ -1279,8 +1314,10 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
Args:
description: Explain why you are running this command in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
command: The bash command to execute. Always use absolute paths for files and directories.
"""
try:
"""
try:
_refresh_skill_capability()
_validate_enabled_skill_paths_in_command(command)
sandbox = ensure_sandbox_initialized(runtime)
if is_local_sandbox(runtime):
if not is_host_bash_allowed():
@@ -1317,17 +1354,20 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
@tool("ls", parse_docstring=True)
def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: str) -> str:
def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: str) -> str:
"""List the contents of a directory up to 2 levels deep in tree format.
Args:
description: Explain why you are listing this directory in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
path: The **absolute** path to the directory to list.
"""
try:
requested_path = path
try:
if _is_skills_path(path):
_refresh_skill_capability()
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
thread_data = None
if is_local_sandbox(runtime):
thread_data = get_thread_data(runtime)
@@ -1364,7 +1404,7 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
@tool("glob", parse_docstring=True)
def glob_tool(
def glob_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
pattern: str,
@@ -1381,10 +1421,13 @@ def glob_tool(
include_dirs: Whether matching directories should also be returned. Default is False.
max_results: Maximum number of paths to return. Default is 200.
"""
try:
requested_path = path
try:
if _is_skills_path(path):
_refresh_skill_capability()
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"glob",
max_results,
@@ -1414,7 +1457,7 @@ def glob_tool(
@tool("grep", parse_docstring=True)
def grep_tool(
def grep_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
pattern: str,
@@ -1435,10 +1478,13 @@ def grep_tool(
case_sensitive: Whether matching is case-sensitive. Default is False.
max_results: Maximum number of matching lines to return. Default is 100.
"""
try:
requested_path = path
try:
if _is_skills_path(path):
_refresh_skill_capability()
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"grep",
max_results,
@@ -1484,7 +1530,7 @@ def grep_tool(
@tool("read_file", parse_docstring=True)
def read_file_tool(
def read_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
path: str,
@@ -1498,18 +1544,30 @@ def read_file_tool(
path: The **absolute** path to the file to read.
start_line: Optional starting line number (1-indexed, inclusive). Use with end_line to read a specific range.
end_line: Optional ending line number (1-indexed, inclusive). Use with start_line to read a specific range.
"""
try:
requested_path = path
if _is_skills_path(requested_path):
from deerflow.tools.builtins.skill_tools import (
"""
try:
requested_path = path
if start_line is not None and (not isinstance(start_line, int) or isinstance(start_line, bool) or start_line < 1):
return "Error: start_line must be a positive integer"
if end_line is not None and (not isinstance(end_line, int) or isinstance(end_line, bool) or end_line < 1):
return "Error: end_line must be a positive integer"
effective_start = start_line or 1
if end_line is not None and end_line < effective_start:
return "Error: end_line must be greater than or equal to start_line"
if _is_skills_path(requested_path):
_refresh_skill_capability()
from deerflow.tools.builtins.skill_tools import (
_resolve_active_agent_id,
can_access_coordinator_only_skill,
is_coordinator_only_skill_path,
)
if is_coordinator_only_skill_path(requested_path) and _resolve_active_agent_id(runtime) != "roundtable-coordinator":
if is_coordinator_only_skill_path(requested_path) and not can_access_coordinator_only_skill(
_resolve_active_agent_id(runtime), "agent_orchestration"
):
# Match skill_view's non-leaking behavior: ordinary agents must not discover whether it exists.
return f"Error: File not found: {requested_path}"
_require_enabled_skill_path(requested_path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
if is_local_sandbox(runtime):
@@ -1522,22 +1580,63 @@ def read_file_tool(
elif not _is_custom_mount_path(path):
path = _resolve_and_validate_user_data_path(path, thread_data)
# Custom mount paths are resolved by LocalSandbox._resolve_path()
content = sandbox.read_file(path)
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
# 非技能路径会被 _record_skill_read 自行忽略。
_record_skill_read(requested_path)
if not content:
return "(empty)"
if start_line is not None and end_line is not None:
content = "\n".join(content.splitlines()[start_line - 1 : end_line])
try:
from deerflow.config.app_config import get_app_config
sandbox_cfg = get_app_config().sandbox
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
except Exception:
max_chars = 50000
return _truncate_read_file_output(content, max_chars)
try:
from deerflow.config.app_config import get_app_config
from deerflow.uploads.policy import get_runtime_large_file_policy
sandbox_cfg = get_app_config().sandbox
configured_max = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
large_file_policy = get_runtime_large_file_policy()
except Exception:
configured_max = 50000
large_file_policy = {
"threshold_bytes": 1024 * 1024,
"max_result_chars": 12000,
"hard_max_result_chars": 32000,
}
policy_max = max(256, int(large_file_policy.get("max_result_chars", 12000)))
hard_max = max(256, min(32000, int(large_file_policy.get("hard_max_result_chars", 32000))))
max_chars = min(policy_max, hard_max, configured_max if configured_max > 0 else hard_max)
# A range-free read of a large local file returns deterministic
# head/middle/tail samples plus whole-file streaming statistics. The
# raw body is never materialized in memory or sent to the model.
if start_line is None and end_line is None and is_local_sandbox(runtime):
try:
physical_path = Path(path)
threshold = max(1, int(large_file_policy.get("threshold_bytes", 1024 * 1024)))
if physical_path.is_file() and physical_path.stat().st_size >= threshold:
from deerflow.uploads.inspection import inspect_text_file
inspected = inspect_text_file(
physical_path,
display_path=requested_path,
max_result_chars=max_chars,
hard_max_result_chars=hard_max,
)
_record_skill_read(requested_path)
return inspected["rendered"]
except OSError:
pass
# Reserve room for a continuation marker so the complete tool result,
# not just its body, remains under the hard cap.
body_budget = max(64, max_chars - 180)
content, next_line, truncated = sandbox.read_file_range(
path,
start_line=effective_start,
end_line=end_line,
max_chars=body_budget,
)
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
# 非技能路径会被 _record_skill_read 自行忽略。
_record_skill_read(requested_path)
if not content and not truncated:
return "(empty)"
if truncated:
continuation = next_line or effective_start
content = f"{content}\n... [bounded read; continue with start_line={continuation}]"
return _truncate_read_file_output(content, max_chars)
except SandboxError as e:
return f"Error: {e}"
except FileNotFoundError: