sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -0,0 +1,346 @@
|
||||
"""Materialize the model-facing, online-only skills filesystem view.
|
||||
|
||||
The canonical skills tree is administration state: it intentionally keeps an
|
||||
offline skill on disk so that it can be brought online again. It therefore
|
||||
must not also be the filesystem capability handed to a model sandbox. This
|
||||
module builds a separate, read-only-at-the-mount-boundary view containing only
|
||||
currently enabled skills.
|
||||
|
||||
The view lives in the writable DeerFlow runtime directory, not below the
|
||||
canonical root (which production commonly mounts read-only). Files are copied,
|
||||
not linked: LocalSandbox executes as the backend user, so a model modifying the
|
||||
disposable view must never mutate canonical skill files through a hard link.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.skills.storage import SkillStorage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SKILL_VIEW_DIR_NAME = ".sandbox-online-skills"
|
||||
SKILL_VIEW_MARKER = ".deerflow-skill-view.json"
|
||||
SKILL_VIEW_PROTOCOL = "deerflow-online-skills-v1"
|
||||
SKILL_SUPPORT_MANIFEST = ".deerflow-skill-support.json"
|
||||
|
||||
_TOKEN_FILE_NAME = ".sandbox-online-skills.token"
|
||||
_LOCK_FILE_NAME = ".sandbox-online-skills.lock"
|
||||
_PROCESS_LOCK = threading.Lock()
|
||||
_BUILTIN_SUPPORT_DEPENDENCIES: dict[tuple[str, str], frozenset[str]] = {
|
||||
("public", "task-report-lib"): frozenset(
|
||||
{
|
||||
"task-report-import",
|
||||
"task-report-build",
|
||||
"task-report-enemy",
|
||||
"task-report-our",
|
||||
"task-report-env",
|
||||
"task-report-judge",
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillSandboxView:
|
||||
"""A refreshed online-only filesystem capability."""
|
||||
|
||||
root: Path
|
||||
token: str
|
||||
enabled_names: frozenset[str]
|
||||
|
||||
@property
|
||||
def marker_content(self) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"protocol": SKILL_VIEW_PROTOCOL,
|
||||
"token": self.token,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
|
||||
def _lock_file(lock_file) -> None:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
lock_file.seek(0, os.SEEK_END)
|
||||
if lock_file.tell() == 0:
|
||||
lock_file.write(b"\0")
|
||||
lock_file.flush()
|
||||
lock_file.seek(0)
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
||||
return
|
||||
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
|
||||
|
||||
def _unlock_file(lock_file) -> None:
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
lock_file.seek(0)
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
return
|
||||
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _view_lock(runtime_root: Path) -> Iterator[None]:
|
||||
"""Serialize view refreshes in this process and across gateway workers."""
|
||||
lock_path = runtime_root / _LOCK_FILE_NAME
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _PROCESS_LOCK, lock_path.open("a+b") as lock_file:
|
||||
_lock_file(lock_file)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_unlock_file(lock_file)
|
||||
|
||||
|
||||
def _load_or_create_token(runtime_root: Path) -> str:
|
||||
token_path = runtime_root / _TOKEN_FILE_NAME
|
||||
try:
|
||||
token = token_path.read_text(encoding="ascii").strip()
|
||||
uuid.UUID(token)
|
||||
return token
|
||||
except (FileNotFoundError, OSError, UnicodeError, ValueError):
|
||||
token = str(uuid.uuid4())
|
||||
token_path.write_text(token + "\n", encoding="ascii")
|
||||
return token
|
||||
|
||||
|
||||
def _support_roots(skills_root: Path, enabled_names: frozenset[str]) -> list[Path]:
|
||||
"""Return explicitly declared internal libraries needed by online skills.
|
||||
|
||||
Cross-skill code is exceptional: ordinary skills are self-contained. A
|
||||
directory without ``SKILL.md`` is included only when it carries a manifest
|
||||
whose ``required_by`` list intersects the enabled skill set. This prevents
|
||||
an arbitrary unregistered directory from becoming model-readable merely by
|
||||
being placed next to a skill.
|
||||
"""
|
||||
roots: list[Path] = []
|
||||
for category in ("public", "custom"):
|
||||
category_root = skills_root / category
|
||||
if not category_root.is_dir():
|
||||
continue
|
||||
for candidate in category_root.iterdir():
|
||||
if not candidate.is_dir() or candidate.is_symlink():
|
||||
continue
|
||||
manifest_path = candidate / SKILL_SUPPORT_MANIFEST
|
||||
dependencies = set(_BUILTIN_SUPPORT_DEPENDENCIES.get((category, candidate.name), ()))
|
||||
if manifest_path.is_file():
|
||||
try:
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
required_by = payload.get("required_by", []) if isinstance(payload, dict) else []
|
||||
if not isinstance(required_by, list):
|
||||
raise ValueError("required_by must be a list")
|
||||
dependencies.update(str(name).strip() for name in required_by if str(name).strip())
|
||||
except (OSError, UnicodeError, ValueError) as exc:
|
||||
logger.warning("Ignoring invalid skill support manifest %s: %s", manifest_path, exc)
|
||||
if dependencies & enabled_names:
|
||||
roots.append(candidate)
|
||||
return roots
|
||||
|
||||
|
||||
def _add_tree(
|
||||
source_root: Path,
|
||||
skills_root: Path,
|
||||
desired_files: dict[Path, Path],
|
||||
desired_dirs: set[Path],
|
||||
) -> None:
|
||||
"""Add regular files below ``source_root`` to the desired view snapshot."""
|
||||
resolved_skills_root = skills_root.resolve()
|
||||
resolved_source = source_root.resolve()
|
||||
try:
|
||||
source_relative = resolved_source.relative_to(resolved_skills_root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"Skill source escapes configured root: {source_root}") from exc
|
||||
|
||||
desired_dirs.add(source_relative)
|
||||
desired_dirs.update(parent for parent in source_relative.parents if parent != Path("."))
|
||||
for current_root, dir_names, file_names in os.walk(resolved_source, followlinks=False):
|
||||
current = Path(current_root)
|
||||
relative_dir = current.relative_to(resolved_skills_root)
|
||||
desired_dirs.add(relative_dir)
|
||||
|
||||
# Symlinks are not copied or followed. Following one could turn a
|
||||
# harmless online skill into a path to an offline/out-of-tree file.
|
||||
retained_dirs: list[str] = []
|
||||
for name in sorted(dir_names):
|
||||
child = current / name
|
||||
if child.is_symlink():
|
||||
logger.warning("Skipping symlink in model-facing skill view: %s", child)
|
||||
continue
|
||||
retained_dirs.append(name)
|
||||
desired_dirs.add(relative_dir / name)
|
||||
dir_names[:] = retained_dirs
|
||||
|
||||
for name in sorted(file_names):
|
||||
source = current / name
|
||||
if source.is_symlink() or not source.is_file():
|
||||
logger.warning("Skipping non-regular file in model-facing skill view: %s", source)
|
||||
continue
|
||||
desired_files[relative_dir / name] = source
|
||||
|
||||
|
||||
def _same_file_snapshot(source: Path, destination: Path) -> bool:
|
||||
try:
|
||||
source_stat = source.stat()
|
||||
destination_stat = destination.stat()
|
||||
except OSError:
|
||||
return False
|
||||
return (
|
||||
source_stat.st_size == destination_stat.st_size
|
||||
and source_stat.st_mtime_ns == destination_stat.st_mtime_ns
|
||||
)
|
||||
|
||||
|
||||
def _copy_file_atomic(source: Path, destination: Path) -> None:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(delete=False, dir=str(destination.parent)) as temp_file:
|
||||
temp_path = Path(temp_file.name)
|
||||
shutil.copy2(source, temp_path)
|
||||
os.replace(temp_path, destination)
|
||||
temp_path = None
|
||||
finally:
|
||||
if temp_path is not None:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _remove_stale_entries(
|
||||
view_root: Path,
|
||||
desired_files: set[Path],
|
||||
desired_dirs: set[Path],
|
||||
) -> None:
|
||||
marker = Path(SKILL_VIEW_MARKER)
|
||||
entries = sorted(
|
||||
view_root.rglob("*"),
|
||||
key=lambda path: len(path.relative_to(view_root).parts),
|
||||
reverse=True,
|
||||
)
|
||||
for entry in entries:
|
||||
relative = entry.relative_to(view_root)
|
||||
if relative == marker:
|
||||
continue
|
||||
if entry.is_symlink():
|
||||
entry.unlink(missing_ok=True)
|
||||
continue
|
||||
if entry.is_dir():
|
||||
if relative not in desired_dirs:
|
||||
shutil.rmtree(entry)
|
||||
continue
|
||||
if relative not in desired_files:
|
||||
entry.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _clear_view(view_root: Path) -> None:
|
||||
if not view_root.exists():
|
||||
return
|
||||
for entry in list(view_root.iterdir()):
|
||||
if entry.is_dir() and not entry.is_symlink():
|
||||
shutil.rmtree(entry)
|
||||
else:
|
||||
entry.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def refresh_skill_sandbox_view(*, storage: SkillStorage | None = None) -> SkillSandboxView:
|
||||
"""Refresh and return the online-only skills view.
|
||||
|
||||
Availability is loaded fresh through ``SkillStorage.load_skills`` on every
|
||||
call. Any failure clears the capability marker and fails closed; callers
|
||||
must not fall back to mounting the canonical skills directory.
|
||||
"""
|
||||
if storage is None:
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
storage = get_or_new_skill_storage()
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
skills_root = Path(storage.get_skills_root_path()).resolve()
|
||||
runtime_root = get_paths().base_dir
|
||||
runtime_root.mkdir(parents=True, exist_ok=True)
|
||||
view_root = runtime_root / SKILL_VIEW_DIR_NAME
|
||||
|
||||
with _view_lock(runtime_root):
|
||||
view_root.mkdir(parents=True, exist_ok=True)
|
||||
marker_path = view_root / SKILL_VIEW_MARKER
|
||||
try:
|
||||
# Ordinary catalogue discovery deliberately degrades open when the
|
||||
# extensions document is unreadable. A model capability must fail
|
||||
# closed, so validate the switch document explicitly and derive
|
||||
# the enabled subset ourselves.
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
|
||||
extensions = ExtensionsConfig.from_file()
|
||||
discovered_skills = storage.load_skills(enabled_only=False)
|
||||
enabled_skills = [
|
||||
skill
|
||||
for skill in discovered_skills
|
||||
if extensions.is_skill_enabled(skill.name, skill.category)
|
||||
]
|
||||
enabled_names = frozenset(skill.name for skill in enabled_skills)
|
||||
token = _load_or_create_token(runtime_root)
|
||||
|
||||
desired_files: dict[Path, Path] = {}
|
||||
desired_dirs: set[Path] = {Path("public"), Path("custom")}
|
||||
for skill in enabled_skills:
|
||||
_add_tree(Path(skill.skill_dir), skills_root, desired_files, desired_dirs)
|
||||
for support_root in _support_roots(skills_root, enabled_names):
|
||||
_add_tree(support_root, skills_root, desired_files, desired_dirs)
|
||||
|
||||
for relative_dir in sorted(desired_dirs, key=lambda path: len(path.parts)):
|
||||
(view_root / relative_dir).mkdir(parents=True, exist_ok=True)
|
||||
for relative, source in desired_files.items():
|
||||
destination = view_root / relative
|
||||
if not _same_file_snapshot(source, destination):
|
||||
_copy_file_atomic(source, destination)
|
||||
|
||||
_remove_stale_entries(view_root, set(desired_files), desired_dirs)
|
||||
view = SkillSandboxView(root=view_root, token=token, enabled_names=enabled_names)
|
||||
marker_path.write_text(view.marker_content, encoding="utf-8")
|
||||
return view
|
||||
except Exception:
|
||||
# Without a marker an AIO sandbox cannot attest that it mounted the
|
||||
# filtered capability. Clear stale content as an additional guard
|
||||
# for LocalSandbox and then propagate the failure.
|
||||
_clear_view(view_root)
|
||||
raise
|
||||
|
||||
|
||||
def expected_skill_view_marker(*, storage: SkillStorage | None = None) -> tuple[Path, str]:
|
||||
"""Refresh the view and return its marker path plus exact expected text."""
|
||||
view = refresh_skill_sandbox_view(storage=storage)
|
||||
return view.root / SKILL_VIEW_MARKER, view.marker_content
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SKILL_SUPPORT_MANIFEST",
|
||||
"SKILL_VIEW_DIR_NAME",
|
||||
"SKILL_VIEW_MARKER",
|
||||
"SKILL_VIEW_PROTOCOL",
|
||||
"SkillSandboxView",
|
||||
"expected_skill_view_marker",
|
||||
"refresh_skill_sandbox_view",
|
||||
]
|
||||
Reference in new issue
Block a user