347 lines
12 KiB
Python
347 lines
12 KiB
Python
"""Materialize the model-facing, online-only skills filesystem view.
|
|
|
|
The canonical skills tree is administration state: it intentionally keeps an
|
|
offline skill on disk so that it can be brought online again. It therefore
|
|
must not also be the filesystem capability handed to a model sandbox. This
|
|
module builds a separate, read-only-at-the-mount-boundary view containing only
|
|
currently enabled skills.
|
|
|
|
The view lives in the writable DeerFlow runtime directory, not below the
|
|
canonical root (which production commonly mounts read-only). Files are copied,
|
|
not linked: LocalSandbox executes as the backend user, so a model modifying the
|
|
disposable view must never mutate canonical skill files through a hard link.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import threading
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from deerflow.skills.storage import SkillStorage
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SKILL_VIEW_DIR_NAME = ".sandbox-online-skills"
|
|
SKILL_VIEW_MARKER = ".deerflow-skill-view.json"
|
|
SKILL_VIEW_PROTOCOL = "deerflow-online-skills-v1"
|
|
SKILL_SUPPORT_MANIFEST = ".deerflow-skill-support.json"
|
|
|
|
_TOKEN_FILE_NAME = ".sandbox-online-skills.token"
|
|
_LOCK_FILE_NAME = ".sandbox-online-skills.lock"
|
|
_PROCESS_LOCK = threading.Lock()
|
|
_BUILTIN_SUPPORT_DEPENDENCIES: dict[tuple[str, str], frozenset[str]] = {
|
|
("public", "task-report-lib"): frozenset(
|
|
{
|
|
"task-report-import",
|
|
"task-report-build",
|
|
"task-report-enemy",
|
|
"task-report-our",
|
|
"task-report-env",
|
|
"task-report-judge",
|
|
}
|
|
),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SkillSandboxView:
|
|
"""A refreshed online-only filesystem capability."""
|
|
|
|
root: Path
|
|
token: str
|
|
enabled_names: frozenset[str]
|
|
|
|
@property
|
|
def marker_content(self) -> str:
|
|
return json.dumps(
|
|
{
|
|
"protocol": SKILL_VIEW_PROTOCOL,
|
|
"token": self.token,
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
|
|
|
|
def _lock_file(lock_file) -> None:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
|
|
lock_file.seek(0, os.SEEK_END)
|
|
if lock_file.tell() == 0:
|
|
lock_file.write(b"\0")
|
|
lock_file.flush()
|
|
lock_file.seek(0)
|
|
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
|
return
|
|
|
|
import fcntl
|
|
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
|
|
|
|
def _unlock_file(lock_file) -> None:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
|
|
lock_file.seek(0)
|
|
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
|
return
|
|
|
|
import fcntl
|
|
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
|
|
|
|
@contextmanager
|
|
def _view_lock(runtime_root: Path) -> Iterator[None]:
|
|
"""Serialize view refreshes in this process and across gateway workers."""
|
|
lock_path = runtime_root / _LOCK_FILE_NAME
|
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with _PROCESS_LOCK, lock_path.open("a+b") as lock_file:
|
|
_lock_file(lock_file)
|
|
try:
|
|
yield
|
|
finally:
|
|
_unlock_file(lock_file)
|
|
|
|
|
|
def _load_or_create_token(runtime_root: Path) -> str:
|
|
token_path = runtime_root / _TOKEN_FILE_NAME
|
|
try:
|
|
token = token_path.read_text(encoding="ascii").strip()
|
|
uuid.UUID(token)
|
|
return token
|
|
except (FileNotFoundError, OSError, UnicodeError, ValueError):
|
|
token = str(uuid.uuid4())
|
|
token_path.write_text(token + "\n", encoding="ascii")
|
|
return token
|
|
|
|
|
|
def _support_roots(skills_root: Path, enabled_names: frozenset[str]) -> list[Path]:
|
|
"""Return explicitly declared internal libraries needed by online skills.
|
|
|
|
Cross-skill code is exceptional: ordinary skills are self-contained. A
|
|
directory without ``SKILL.md`` is included only when it carries a manifest
|
|
whose ``required_by`` list intersects the enabled skill set. This prevents
|
|
an arbitrary unregistered directory from becoming model-readable merely by
|
|
being placed next to a skill.
|
|
"""
|
|
roots: list[Path] = []
|
|
for category in ("public", "custom"):
|
|
category_root = skills_root / category
|
|
if not category_root.is_dir():
|
|
continue
|
|
for candidate in category_root.iterdir():
|
|
if not candidate.is_dir() or candidate.is_symlink():
|
|
continue
|
|
manifest_path = candidate / SKILL_SUPPORT_MANIFEST
|
|
dependencies = set(_BUILTIN_SUPPORT_DEPENDENCIES.get((category, candidate.name), ()))
|
|
if manifest_path.is_file():
|
|
try:
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
required_by = payload.get("required_by", []) if isinstance(payload, dict) else []
|
|
if not isinstance(required_by, list):
|
|
raise ValueError("required_by must be a list")
|
|
dependencies.update(str(name).strip() for name in required_by if str(name).strip())
|
|
except (OSError, UnicodeError, ValueError) as exc:
|
|
logger.warning("Ignoring invalid skill support manifest %s: %s", manifest_path, exc)
|
|
if dependencies & enabled_names:
|
|
roots.append(candidate)
|
|
return roots
|
|
|
|
|
|
def _add_tree(
|
|
source_root: Path,
|
|
skills_root: Path,
|
|
desired_files: dict[Path, Path],
|
|
desired_dirs: set[Path],
|
|
) -> None:
|
|
"""Add regular files below ``source_root`` to the desired view snapshot."""
|
|
resolved_skills_root = skills_root.resolve()
|
|
resolved_source = source_root.resolve()
|
|
try:
|
|
source_relative = resolved_source.relative_to(resolved_skills_root)
|
|
except ValueError as exc:
|
|
raise RuntimeError(f"Skill source escapes configured root: {source_root}") from exc
|
|
|
|
desired_dirs.add(source_relative)
|
|
desired_dirs.update(parent for parent in source_relative.parents if parent != Path("."))
|
|
for current_root, dir_names, file_names in os.walk(resolved_source, followlinks=False):
|
|
current = Path(current_root)
|
|
relative_dir = current.relative_to(resolved_skills_root)
|
|
desired_dirs.add(relative_dir)
|
|
|
|
# Symlinks are not copied or followed. Following one could turn a
|
|
# harmless online skill into a path to an offline/out-of-tree file.
|
|
retained_dirs: list[str] = []
|
|
for name in sorted(dir_names):
|
|
child = current / name
|
|
if child.is_symlink():
|
|
logger.warning("Skipping symlink in model-facing skill view: %s", child)
|
|
continue
|
|
retained_dirs.append(name)
|
|
desired_dirs.add(relative_dir / name)
|
|
dir_names[:] = retained_dirs
|
|
|
|
for name in sorted(file_names):
|
|
source = current / name
|
|
if source.is_symlink() or not source.is_file():
|
|
logger.warning("Skipping non-regular file in model-facing skill view: %s", source)
|
|
continue
|
|
desired_files[relative_dir / name] = source
|
|
|
|
|
|
def _same_file_snapshot(source: Path, destination: Path) -> bool:
|
|
try:
|
|
source_stat = source.stat()
|
|
destination_stat = destination.stat()
|
|
except OSError:
|
|
return False
|
|
return (
|
|
source_stat.st_size == destination_stat.st_size
|
|
and source_stat.st_mtime_ns == destination_stat.st_mtime_ns
|
|
)
|
|
|
|
|
|
def _copy_file_atomic(source: Path, destination: Path) -> None:
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
temp_path: Path | None = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(delete=False, dir=str(destination.parent)) as temp_file:
|
|
temp_path = Path(temp_file.name)
|
|
shutil.copy2(source, temp_path)
|
|
os.replace(temp_path, destination)
|
|
temp_path = None
|
|
finally:
|
|
if temp_path is not None:
|
|
temp_path.unlink(missing_ok=True)
|
|
|
|
|
|
def _remove_stale_entries(
|
|
view_root: Path,
|
|
desired_files: set[Path],
|
|
desired_dirs: set[Path],
|
|
) -> None:
|
|
marker = Path(SKILL_VIEW_MARKER)
|
|
entries = sorted(
|
|
view_root.rglob("*"),
|
|
key=lambda path: len(path.relative_to(view_root).parts),
|
|
reverse=True,
|
|
)
|
|
for entry in entries:
|
|
relative = entry.relative_to(view_root)
|
|
if relative == marker:
|
|
continue
|
|
if entry.is_symlink():
|
|
entry.unlink(missing_ok=True)
|
|
continue
|
|
if entry.is_dir():
|
|
if relative not in desired_dirs:
|
|
shutil.rmtree(entry)
|
|
continue
|
|
if relative not in desired_files:
|
|
entry.unlink(missing_ok=True)
|
|
|
|
|
|
def _clear_view(view_root: Path) -> None:
|
|
if not view_root.exists():
|
|
return
|
|
for entry in list(view_root.iterdir()):
|
|
if entry.is_dir() and not entry.is_symlink():
|
|
shutil.rmtree(entry)
|
|
else:
|
|
entry.unlink(missing_ok=True)
|
|
|
|
|
|
def refresh_skill_sandbox_view(*, storage: SkillStorage | None = None) -> SkillSandboxView:
|
|
"""Refresh and return the online-only skills view.
|
|
|
|
Availability is loaded fresh through ``SkillStorage.load_skills`` on every
|
|
call. Any failure clears the capability marker and fails closed; callers
|
|
must not fall back to mounting the canonical skills directory.
|
|
"""
|
|
if storage is None:
|
|
from deerflow.skills.storage import get_or_new_skill_storage
|
|
|
|
storage = get_or_new_skill_storage()
|
|
|
|
from deerflow.config.paths import get_paths
|
|
|
|
skills_root = Path(storage.get_skills_root_path()).resolve()
|
|
runtime_root = get_paths().base_dir
|
|
runtime_root.mkdir(parents=True, exist_ok=True)
|
|
view_root = runtime_root / SKILL_VIEW_DIR_NAME
|
|
|
|
with _view_lock(runtime_root):
|
|
view_root.mkdir(parents=True, exist_ok=True)
|
|
marker_path = view_root / SKILL_VIEW_MARKER
|
|
try:
|
|
# Ordinary catalogue discovery deliberately degrades open when the
|
|
# extensions document is unreadable. A model capability must fail
|
|
# closed, so validate the switch document explicitly and derive
|
|
# the enabled subset ourselves.
|
|
from deerflow.config.extensions_config import ExtensionsConfig
|
|
|
|
extensions = ExtensionsConfig.from_file()
|
|
discovered_skills = storage.load_skills(enabled_only=False)
|
|
enabled_skills = [
|
|
skill
|
|
for skill in discovered_skills
|
|
if extensions.is_skill_enabled(skill.name, skill.category)
|
|
]
|
|
enabled_names = frozenset(skill.name for skill in enabled_skills)
|
|
token = _load_or_create_token(runtime_root)
|
|
|
|
desired_files: dict[Path, Path] = {}
|
|
desired_dirs: set[Path] = {Path("public"), Path("custom")}
|
|
for skill in enabled_skills:
|
|
_add_tree(Path(skill.skill_dir), skills_root, desired_files, desired_dirs)
|
|
for support_root in _support_roots(skills_root, enabled_names):
|
|
_add_tree(support_root, skills_root, desired_files, desired_dirs)
|
|
|
|
for relative_dir in sorted(desired_dirs, key=lambda path: len(path.parts)):
|
|
(view_root / relative_dir).mkdir(parents=True, exist_ok=True)
|
|
for relative, source in desired_files.items():
|
|
destination = view_root / relative
|
|
if not _same_file_snapshot(source, destination):
|
|
_copy_file_atomic(source, destination)
|
|
|
|
_remove_stale_entries(view_root, set(desired_files), desired_dirs)
|
|
view = SkillSandboxView(root=view_root, token=token, enabled_names=enabled_names)
|
|
marker_path.write_text(view.marker_content, encoding="utf-8")
|
|
return view
|
|
except Exception:
|
|
# Without a marker an AIO sandbox cannot attest that it mounted the
|
|
# filtered capability. Clear stale content as an additional guard
|
|
# for LocalSandbox and then propagate the failure.
|
|
_clear_view(view_root)
|
|
raise
|
|
|
|
|
|
def expected_skill_view_marker(*, storage: SkillStorage | None = None) -> tuple[Path, str]:
|
|
"""Refresh the view and return its marker path plus exact expected text."""
|
|
view = refresh_skill_sandbox_view(storage=storage)
|
|
return view.root / SKILL_VIEW_MARKER, view.marker_content
|
|
|
|
|
|
__all__ = [
|
|
"SKILL_SUPPORT_MANIFEST",
|
|
"SKILL_VIEW_DIR_NAME",
|
|
"SKILL_VIEW_MARKER",
|
|
"SKILL_VIEW_PROTOCOL",
|
|
"SkillSandboxView",
|
|
"expected_skill_view_marker",
|
|
"refresh_skill_sandbox_view",
|
|
]
|