"""Materialize the model-facing, online-only skills filesystem view. The canonical skills tree is administration state: it intentionally keeps an offline skill on disk so that it can be brought online again. It therefore must not also be the filesystem capability handed to a model sandbox. This module builds a separate, read-only-at-the-mount-boundary view containing only currently enabled skills. The view lives in the writable DeerFlow runtime directory, not below the canonical root (which production commonly mounts read-only). Files are copied, not linked: LocalSandbox executes as the backend user, so a model modifying the disposable view must never mutate canonical skill files through a hard link. """ from __future__ import annotations import json import logging import os import shutil import tempfile import threading import uuid from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: from deerflow.skills.storage import SkillStorage logger = logging.getLogger(__name__) SKILL_VIEW_DIR_NAME = ".sandbox-online-skills" SKILL_VIEW_MARKER = ".deerflow-skill-view.json" SKILL_VIEW_PROTOCOL = "deerflow-online-skills-v1" SKILL_SUPPORT_MANIFEST = ".deerflow-skill-support.json" _TOKEN_FILE_NAME = ".sandbox-online-skills.token" _LOCK_FILE_NAME = ".sandbox-online-skills.lock" _PROCESS_LOCK = threading.Lock() _BUILTIN_SUPPORT_DEPENDENCIES: dict[tuple[str, str], frozenset[str]] = { ("public", "task-report-lib"): frozenset( { "task-report-import", "task-report-build", "task-report-enemy", "task-report-our", "task-report-env", "task-report-judge", } ), } @dataclass(frozen=True) class SkillSandboxView: """A refreshed online-only filesystem capability.""" root: Path token: str enabled_names: frozenset[str] @property def marker_content(self) -> str: return json.dumps( { "protocol": SKILL_VIEW_PROTOCOL, "token": self.token, }, sort_keys=True, ) def _lock_file(lock_file) -> None: if os.name == "nt": import msvcrt lock_file.seek(0, os.SEEK_END) if lock_file.tell() == 0: lock_file.write(b"\0") lock_file.flush() lock_file.seek(0) msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) return import fcntl fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) def _unlock_file(lock_file) -> None: if os.name == "nt": import msvcrt lock_file.seek(0) msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) return import fcntl fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) @contextmanager def _view_lock(runtime_root: Path) -> Iterator[None]: """Serialize view refreshes in this process and across gateway workers.""" lock_path = runtime_root / _LOCK_FILE_NAME lock_path.parent.mkdir(parents=True, exist_ok=True) with _PROCESS_LOCK, lock_path.open("a+b") as lock_file: _lock_file(lock_file) try: yield finally: _unlock_file(lock_file) def _load_or_create_token(runtime_root: Path) -> str: token_path = runtime_root / _TOKEN_FILE_NAME try: token = token_path.read_text(encoding="ascii").strip() uuid.UUID(token) return token except (FileNotFoundError, OSError, UnicodeError, ValueError): token = str(uuid.uuid4()) token_path.write_text(token + "\n", encoding="ascii") return token def _support_roots(skills_root: Path, enabled_names: frozenset[str]) -> list[Path]: """Return explicitly declared internal libraries needed by online skills. Cross-skill code is exceptional: ordinary skills are self-contained. A directory without ``SKILL.md`` is included only when it carries a manifest whose ``required_by`` list intersects the enabled skill set. This prevents an arbitrary unregistered directory from becoming model-readable merely by being placed next to a skill. """ roots: list[Path] = [] for category in ("public", "custom"): category_root = skills_root / category if not category_root.is_dir(): continue for candidate in category_root.iterdir(): if not candidate.is_dir() or candidate.is_symlink(): continue manifest_path = candidate / SKILL_SUPPORT_MANIFEST dependencies = set(_BUILTIN_SUPPORT_DEPENDENCIES.get((category, candidate.name), ())) if manifest_path.is_file(): try: payload = json.loads(manifest_path.read_text(encoding="utf-8")) required_by = payload.get("required_by", []) if isinstance(payload, dict) else [] if not isinstance(required_by, list): raise ValueError("required_by must be a list") dependencies.update(str(name).strip() for name in required_by if str(name).strip()) except (OSError, UnicodeError, ValueError) as exc: logger.warning("Ignoring invalid skill support manifest %s: %s", manifest_path, exc) if dependencies & enabled_names: roots.append(candidate) return roots def _add_tree( source_root: Path, skills_root: Path, desired_files: dict[Path, Path], desired_dirs: set[Path], ) -> None: """Add regular files below ``source_root`` to the desired view snapshot.""" resolved_skills_root = skills_root.resolve() resolved_source = source_root.resolve() try: source_relative = resolved_source.relative_to(resolved_skills_root) except ValueError as exc: raise RuntimeError(f"Skill source escapes configured root: {source_root}") from exc desired_dirs.add(source_relative) desired_dirs.update(parent for parent in source_relative.parents if parent != Path(".")) for current_root, dir_names, file_names in os.walk(resolved_source, followlinks=False): current = Path(current_root) relative_dir = current.relative_to(resolved_skills_root) desired_dirs.add(relative_dir) # Symlinks are not copied or followed. Following one could turn a # harmless online skill into a path to an offline/out-of-tree file. retained_dirs: list[str] = [] for name in sorted(dir_names): child = current / name if child.is_symlink(): logger.warning("Skipping symlink in model-facing skill view: %s", child) continue retained_dirs.append(name) desired_dirs.add(relative_dir / name) dir_names[:] = retained_dirs for name in sorted(file_names): source = current / name if source.is_symlink() or not source.is_file(): logger.warning("Skipping non-regular file in model-facing skill view: %s", source) continue desired_files[relative_dir / name] = source def _same_file_snapshot(source: Path, destination: Path) -> bool: try: source_stat = source.stat() destination_stat = destination.stat() except OSError: return False return ( source_stat.st_size == destination_stat.st_size and source_stat.st_mtime_ns == destination_stat.st_mtime_ns ) def _copy_file_atomic(source: Path, destination: Path) -> None: destination.parent.mkdir(parents=True, exist_ok=True) temp_path: Path | None = None try: with tempfile.NamedTemporaryFile(delete=False, dir=str(destination.parent)) as temp_file: temp_path = Path(temp_file.name) shutil.copy2(source, temp_path) os.replace(temp_path, destination) temp_path = None finally: if temp_path is not None: temp_path.unlink(missing_ok=True) def _remove_stale_entries( view_root: Path, desired_files: set[Path], desired_dirs: set[Path], ) -> None: marker = Path(SKILL_VIEW_MARKER) entries = sorted( view_root.rglob("*"), key=lambda path: len(path.relative_to(view_root).parts), reverse=True, ) for entry in entries: relative = entry.relative_to(view_root) if relative == marker: continue if entry.is_symlink(): entry.unlink(missing_ok=True) continue if entry.is_dir(): if relative not in desired_dirs: shutil.rmtree(entry) continue if relative not in desired_files: entry.unlink(missing_ok=True) def _clear_view(view_root: Path) -> None: if not view_root.exists(): return for entry in list(view_root.iterdir()): if entry.is_dir() and not entry.is_symlink(): shutil.rmtree(entry) else: entry.unlink(missing_ok=True) def refresh_skill_sandbox_view(*, storage: SkillStorage | None = None) -> SkillSandboxView: """Refresh and return the online-only skills view. Availability is loaded fresh through ``SkillStorage.load_skills`` on every call. Any failure clears the capability marker and fails closed; callers must not fall back to mounting the canonical skills directory. """ if storage is None: from deerflow.skills.storage import get_or_new_skill_storage storage = get_or_new_skill_storage() from deerflow.config.paths import get_paths skills_root = Path(storage.get_skills_root_path()).resolve() runtime_root = get_paths().base_dir runtime_root.mkdir(parents=True, exist_ok=True) view_root = runtime_root / SKILL_VIEW_DIR_NAME with _view_lock(runtime_root): view_root.mkdir(parents=True, exist_ok=True) marker_path = view_root / SKILL_VIEW_MARKER try: # Ordinary catalogue discovery deliberately degrades open when the # extensions document is unreadable. A model capability must fail # closed, so validate the switch document explicitly and derive # the enabled subset ourselves. from deerflow.config.extensions_config import ExtensionsConfig extensions = ExtensionsConfig.from_file() discovered_skills = storage.load_skills(enabled_only=False) enabled_skills = [ skill for skill in discovered_skills if extensions.is_skill_enabled(skill.name, skill.category) ] enabled_names = frozenset(skill.name for skill in enabled_skills) token = _load_or_create_token(runtime_root) desired_files: dict[Path, Path] = {} desired_dirs: set[Path] = {Path("public"), Path("custom")} for skill in enabled_skills: _add_tree(Path(skill.skill_dir), skills_root, desired_files, desired_dirs) for support_root in _support_roots(skills_root, enabled_names): _add_tree(support_root, skills_root, desired_files, desired_dirs) for relative_dir in sorted(desired_dirs, key=lambda path: len(path.parts)): (view_root / relative_dir).mkdir(parents=True, exist_ok=True) for relative, source in desired_files.items(): destination = view_root / relative if not _same_file_snapshot(source, destination): _copy_file_atomic(source, destination) _remove_stale_entries(view_root, set(desired_files), desired_dirs) view = SkillSandboxView(root=view_root, token=token, enabled_names=enabled_names) marker_path.write_text(view.marker_content, encoding="utf-8") return view except Exception: # Without a marker an AIO sandbox cannot attest that it mounted the # filtered capability. Clear stale content as an additional guard # for LocalSandbox and then propagate the failure. _clear_view(view_root) raise def expected_skill_view_marker(*, storage: SkillStorage | None = None) -> tuple[Path, str]: """Refresh the view and return its marker path plus exact expected text.""" view = refresh_skill_sandbox_view(storage=storage) return view.root / SKILL_VIEW_MARKER, view.marker_content __all__ = [ "SKILL_SUPPORT_MANIFEST", "SKILL_VIEW_DIR_NAME", "SKILL_VIEW_MARKER", "SKILL_VIEW_PROTOCOL", "SkillSandboxView", "expected_skill_view_marker", "refresh_skill_sandbox_view", ]