sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -1,14 +1,21 @@
import json
import posixpath
import re
import shlex
from pathlib import Path
import time
from collections.abc import Callable
from functools import wraps
from inspect import signature
from pathlib import Path, PurePosixPath
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from pydantic import BaseModel, ConfigDict, Field
from deerflow.agents.thread_state import ThreadDataState, ThreadState
from deerflow.config import get_app_config
from deerflow.config.paths import VIRTUAL_PATH_PREFIX
from deerflow.html_runtime.html_postprocess import maybe_fix_generated_html
from deerflow.sandbox.exceptions import (
SandboxError,
SandboxNotFoundError,
@@ -19,6 +26,7 @@ from deerflow.sandbox.sandbox import Sandbox
from deerflow.sandbox.sandbox_provider import get_sandbox_provider
from deerflow.sandbox.search import GrepMatch
from deerflow.sandbox.security import LOCAL_HOST_BASH_DISABLED_MESSAGE, is_host_bash_allowed
from deerflow.sandbox.skill_view_scope import get_active_skill_view_scope, with_skill_view_scope
_ABSOLUTE_PATH_PATTERN = re.compile(r"(?<![:\w])(?<!:/)/(?:[^\s\"'`;&|<>()]+)")
_FILE_URL_PATTERN = re.compile(r"\bfile://\S+", re.IGNORECASE)
@@ -99,36 +107,189 @@ def _get_skills_container_path() -> str:
def _get_skills_host_path() -> str | None:
"""Get the skills host filesystem path from config.
"""Get the online-only model-facing skills path.
Returns None if the skills directory does not exist or config cannot be
loaded. Only successful lookups are cached; failures are retried on the
next call so that a transiently unavailable skills directory does not
permanently disable skills access.
Never return the canonical administrative skills tree. Offline skills
intentionally remain there and would otherwise be reachable from bash by
constructing a path dynamically instead of spelling it in the tool call.
A warm cache resolves the published view without stating every skill file.
"""
cached = getattr(_get_skills_host_path, "_cached", None)
if cached is not None:
return cached
try:
from deerflow.config import get_app_config
from deerflow.sandbox.skill_view import current_skill_view_root
config = get_app_config()
skills_path = config.skills.get_skills_path()
if skills_path.exists():
value = str(skills_path)
_get_skills_host_path._cached = value # type: ignore[attr-defined]
return value
return str(current_skill_view_root())
except Exception:
pass
return None
def _refresh_skill_capability() -> None:
"""Refresh the filesystem capability used by local and AIO sandboxes."""
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
refresh_skill_sandbox_view()
def _prepare_skill_paths(paths: list[str]) -> None:
"""Refresh only the skills a tool is about to read, when the cache is on."""
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
if not _skill_view_cache_enabled():
_refresh_skill_capability()
return
prepare_skill_tool_view(paths, container_path=_get_skills_container_path())
def _prepare_command_skill_capability(command: str) -> None:
"""Skip the skill-view check when a command does not name a skill.
The online view is still the security boundary. Management writes remove
an offlined skill from that view immediately, so a later command that
hides the path cannot run it. Commands that spell a skill path reconcile
only that skill.
"""
from deerflow.sandbox.skill_view import _skill_view_cache_enabled, prepare_skill_tool_view
if not _skill_view_cache_enabled():
_refresh_skill_capability()
return
skills_container = _get_skills_container_path()
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
if boundary.search(command) is None:
from deerflow.sandbox.skill_view import note_skill_view_check
note_skill_view_check(status="skipped")
return
paths: list[str] = []
url_spans = _non_file_url_spans(command)
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
if _is_in_spans(match.start(), url_spans):
continue
path = match.group()
if _is_skills_path(path):
paths.append(path)
prepare_skill_tool_view(paths, container_path=skills_container)
def _skill_paths_in_text(text: str) -> list[str]:
paths: list[str] = []
for match in _ABSOLUTE_PATH_PATTERN.finditer(text):
path = match.group()
if _is_skills_path(path) and path not in paths:
paths.append(path)
return paths
def _finish_skill_view_trace(tool: str, command: str, started: float, *, always: bool) -> None:
"""Write the diagnostic row only after this tool call has returned."""
try:
from deerflow.sandbox.skill_view import commit_skill_view_trace, skill_names_for_paths
commit_skill_view_trace(
tool=tool,
command=command,
skill_names=skill_names_for_paths(_skill_paths_in_text(command), container_path=_get_skills_container_path()),
tool_ms=(time.perf_counter() - started) * 1000,
allow_empty=always,
)
except Exception:
return
def _trace_completed_tool(tool_name: str, *, command_from: str, always: bool = False) -> Callable:
"""Publish one timing row after the wrapped tool returns, including on error."""
def decorator[**P, R](fn: Callable[P, R]) -> Callable[P, R]:
sig = signature(fn)
@wraps(fn)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
from deerflow.sandbox.skill_view import begin_skill_view_trace, end_skill_view_trace
bound = sig.bind_partial(*args, **kwargs)
command = str(bound.arguments.get(command_from) or "")
if tool_name in {"glob", "grep"}:
command = f"{bound.arguments.get('pattern') or ''} {command}".strip()
started = time.perf_counter()
token = begin_skill_view_trace()
try:
return fn(*args, **kwargs)
finally:
try:
_finish_skill_view_trace(tool_name, command, started, always=always)
finally:
end_skill_view_trace(token)
wrapper.__signature__ = sig # type: ignore[attr-defined]
return wrapper
return decorator
def _is_skills_path(path: str) -> bool:
"""Check if a path is under the skills container path."""
skills_prefix = _get_skills_container_path()
return path == skills_prefix or path.startswith(f"{skills_prefix}/")
def _require_enabled_skill_path(path: str) -> None:
"""Fail closed unless ``path`` belongs to a currently enabled skill."""
scope = get_active_skill_view_scope()
if scope is not None:
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
# The entry snapshot already validated availability and discovered the
# real skill directories. Re-parsing the full catalogue here (and again
# during local path validation) would defeat per-call view reuse.
view = refresh_skill_sandbox_view()
prefix = _get_skills_container_path().replace("\\", "/").rstrip("/")
normalized = path.replace("\\", "/").rstrip("/")
if ".." in PurePosixPath(normalized).parts:
raise PermissionError("Skill path traversal is not allowed")
for _name, category, relative in view.enabled_roots:
# Root/category-wide access is never a model capability.
if not relative or relative == ".":
continue
skill_root = f"{prefix}/{category}/{relative}"
if normalized == skill_root or normalized.startswith(f"{skill_root}/"):
return
raise PermissionError("Skill path is unavailable")
from deerflow.tools.builtins.skill_tools import require_enabled_skill_path
require_enabled_skill_path(path, container_path=_get_skills_container_path())
def _validate_enabled_skill_paths_in_command(command: str) -> None:
"""Give a clear error for explicit unavailable skill paths.
This is a usability policy, not the security boundary. Shell expansion
can hide paths from string inspection, so the enforceable boundary is the
online-only filesystem view. Management writes update that view immediately;
a command that names a skill reconciles that skill before it runs.
"""
skills_container = _get_skills_container_path()
boundary = re.compile(rf"{re.escape(skills_container)}(?=$|[/\s\"'`;&|<>()])")
if boundary.search(command) is None:
return
matched = False
url_spans = _non_file_url_spans(command)
for match in _ABSOLUTE_PATH_PATTERN.finditer(command):
if _is_in_spans(match.start(), url_spans):
continue
path = match.group()
if not _is_skills_path(path):
continue
matched = True
_require_enabled_skill_path(path)
# Quoting/concatenation that hides the concrete skill directory is not a
# valid way around the gate. Exact, discoverable paths are required.
if not matched:
raise PermissionError("Skill path is unavailable")
def _resolve_skills_path(path: str) -> str:
"""Resolve a virtual skills path to a host filesystem path.
@@ -649,6 +810,7 @@ def validate_local_tool_path(path: str, thread_data: ThreadDataState | None, *,
# Skills paths — read-only access only
if _is_skills_path(path):
_require_enabled_skill_path(path)
if not read_only:
raise PermissionError(f"Write access to skills path is not allowed: {path}")
return
@@ -1268,6 +1430,8 @@ def _truncate_ls_output(output: str, max_chars: int) -> str:
@tool("bash", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("bash", command_from="command", always=True)
def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, command: str) -> str:
"""Execute a bash command in a Linux environment.
@@ -1281,6 +1445,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
command: The bash command to execute. Always use absolute paths for files and directories.
"""
try:
_prepare_command_skill_capability(command)
_validate_enabled_skill_paths_in_command(command)
sandbox = ensure_sandbox_initialized(runtime)
if is_local_sandbox(runtime):
if not is_host_bash_allowed():
@@ -1317,6 +1483,8 @@ def bash_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, com
@tool("ls", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("ls", command_from="path")
def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path: str) -> str:
"""List the contents of a directory up to 2 levels deep in tree format.
@@ -1324,10 +1492,13 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
description: Explain why you are listing this directory in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
path: The **absolute** path to the directory to list.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
thread_data = None
if is_local_sandbox(runtime):
thread_data = get_thread_data(runtime)
@@ -1364,6 +1535,8 @@ def ls_tool(runtime: ToolRuntime[ContextT, ThreadState], description: str, path:
@tool("glob", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("glob", command_from="path")
def glob_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1381,10 +1554,13 @@ def glob_tool(
include_dirs: Whether matching directories should also be returned. Default is False.
max_results: Maximum number of paths to return. Default is 200.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"glob",
max_results,
@@ -1414,6 +1590,8 @@ def glob_tool(
@tool("grep", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("grep", command_from="path")
def grep_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1435,10 +1613,13 @@ def grep_tool(
case_sensitive: Whether matching is case-sensitive. Default is False.
max_results: Maximum number of matching lines to return. Default is 100.
"""
requested_path = path
try:
if _is_skills_path(path):
_prepare_skill_paths([path])
_require_enabled_skill_path(path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
requested_path = path
effective_max_results = _resolve_max_results(
"grep",
max_results,
@@ -1484,6 +1665,8 @@ def grep_tool(
@tool("read_file", parse_docstring=True)
@with_skill_view_scope
@_trace_completed_tool("read_file", command_from="path")
def read_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1501,15 +1684,27 @@ def read_file_tool(
"""
try:
requested_path = path
if start_line is not None and (not isinstance(start_line, int) or isinstance(start_line, bool) or start_line < 1):
return "Error: start_line must be a positive integer"
if end_line is not None and (not isinstance(end_line, int) or isinstance(end_line, bool) or end_line < 1):
return "Error: end_line must be a positive integer"
effective_start = start_line or 1
if end_line is not None and end_line < effective_start:
return "Error: end_line must be greater than or equal to start_line"
if _is_skills_path(requested_path):
_prepare_skill_paths([requested_path])
from deerflow.tools.builtins.skill_tools import (
_resolve_active_agent_id,
can_access_coordinator_only_skill,
is_coordinator_only_skill_path,
)
if is_coordinator_only_skill_path(requested_path) and _resolve_active_agent_id(runtime) != "roundtable-coordinator":
if is_coordinator_only_skill_path(requested_path) and not can_access_coordinator_only_skill(
_resolve_active_agent_id(runtime), "agent_orchestration"
):
# Match skill_view's non-leaking behavior: ordinary agents must not discover whether it exists.
return f"Error: File not found: {requested_path}"
_require_enabled_skill_path(requested_path)
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
if is_local_sandbox(runtime):
@@ -1522,21 +1717,62 @@ def read_file_tool(
elif not _is_custom_mount_path(path):
path = _resolve_and_validate_user_data_path(path, thread_data)
# Custom mount paths are resolved by LocalSandbox._resolve_path()
content = sandbox.read_file(path)
try:
from deerflow.config.app_config import get_app_config
from deerflow.uploads.policy import get_runtime_large_file_policy
sandbox_cfg = get_app_config().sandbox
configured_max = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
large_file_policy = get_runtime_large_file_policy()
except Exception:
configured_max = 50000
large_file_policy = {
"threshold_bytes": 1024 * 1024,
"max_result_chars": 12000,
"hard_max_result_chars": 32000,
}
policy_max = max(256, int(large_file_policy.get("max_result_chars", 12000)))
hard_max = max(256, min(32000, int(large_file_policy.get("hard_max_result_chars", 32000))))
max_chars = min(policy_max, hard_max, configured_max if configured_max > 0 else hard_max)
# A range-free read of a large local file returns deterministic
# head/middle/tail samples plus whole-file streaming statistics. The
# raw body is never materialized in memory or sent to the model.
if start_line is None and end_line is None and is_local_sandbox(runtime):
try:
physical_path = Path(path)
threshold = max(1, int(large_file_policy.get("threshold_bytes", 1024 * 1024)))
if physical_path.is_file() and physical_path.stat().st_size >= threshold:
from deerflow.uploads.inspection import inspect_text_file
inspected = inspect_text_file(
physical_path,
display_path=requested_path,
max_result_chars=max_chars,
hard_max_result_chars=hard_max,
)
_record_skill_read(requested_path)
return inspected["rendered"]
except OSError:
pass
# Reserve room for a continuation marker so the complete tool result,
# not just its body, remains under the hard cap.
body_budget = max(64, max_chars - 180)
content, next_line, truncated = sandbox.read_file_range(
path,
start_line=effective_start,
end_line=end_line,
max_chars=body_budget,
)
# 读成功即记一次技能使用 —— requested_path 仍是虚拟路径(本地/容器沙箱都一样),
# 非技能路径会被 _record_skill_read 自行忽略。
_record_skill_read(requested_path)
if not content:
if not content and not truncated:
return "(empty)"
if start_line is not None and end_line is not None:
content = "\n".join(content.splitlines()[start_line - 1 : end_line])
try:
from deerflow.config.app_config import get_app_config
sandbox_cfg = get_app_config().sandbox
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
except Exception:
max_chars = 50000
if truncated:
continuation = next_line or effective_start
content = f"{content}\n... [bounded read; continue with start_line={continuation}]"
return _truncate_read_file_output(content, max_chars)
except SandboxError as e:
return f"Error: {e}"
@@ -1551,11 +1787,12 @@ def read_file_tool(
@tool("write_file", parse_docstring=True)
@with_skill_view_scope
def write_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
path: str,
content: str,
description: str = "",
path: str = "",
content: str = "",
append: bool = False,
) -> str:
"""Write text content to a file.
@@ -1565,6 +1802,13 @@ def write_file_tool(
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
content: The content to write to the file. ALWAYS PROVIDE THIS PARAMETER THIRD.
"""
if not str(path).strip():
return "Error: write_file requires an absolute path."
if content is None or content == "":
return (
"Error: write_file requires the full file content in the same call as path. "
"Retry once with path and content together."
)
try:
sandbox = ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
@@ -1576,6 +1820,7 @@ def write_file_tool(
path = _resolve_and_validate_user_data_path(path, thread_data)
# Custom mount paths are resolved by LocalSandbox._resolve_path()
with get_file_operation_lock(sandbox, path):
content = maybe_fix_generated_html(path, content)
sandbox.write_file(path, content, append)
return "OK"
except SandboxError as e:
@@ -1590,7 +1835,90 @@ def write_file_tool(
return f"Error: Unexpected error writing file: {_sanitize_error(e, runtime)}"
class _BeginWriteFileInput(BaseModel):
"""Small, strict prepare envelope; file bytes are never valid here."""
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
runtime: ToolRuntime[ContextT, ThreadState]
description: str = Field(
default="",
max_length=240,
description="A short explanation of the file being written.",
)
path: str = Field(
min_length=1,
max_length=4096,
description="The absolute sandbox path of the target file.",
)
append: bool = Field(
default=False,
description="Whether the final write appends instead of overwriting.",
)
@tool(
"begin_write_file",
args_schema=_BeginWriteFileInput,
parse_docstring=True,
)
@with_skill_view_scope
def begin_write_file_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str = "",
path: str = "",
append: bool = False,
) -> str:
"""Prepare a file target for a streamed text-body write without writing it yet.
This internal compatibility handshake is exposed only to models configured
with ``write_file_transport: artifact_renderer`` (or the legacy
``text_bridge`` mode). It validates the target but writes no bytes. After a
``status=ready`` result, DeerFlow obtains the body through the configured
transport and executes the real ``write_file`` tool only after validation.
Args:
description: Explain why you are writing to this file in short words. ALWAYS PROVIDE THIS PARAMETER FIRST.
path: The **absolute** path to the file to write to. ALWAYS PROVIDE THIS PARAMETER SECOND.
append: Whether the final write should append instead of overwrite.
"""
if not str(path).strip().startswith("/"):
return "Error: begin_write_file requires an absolute path."
requested_path = path
try:
# Match write_file's sandbox acquisition and local path authorization,
# but deliberately do not take a file lock or persist any bytes.
ensure_sandbox_initialized(runtime)
ensure_thread_directories_exist(runtime)
if is_local_sandbox(runtime):
thread_data = get_thread_data(runtime)
validate_local_tool_path(path, thread_data)
if not _is_custom_mount_path(path):
_resolve_and_validate_user_data_path(path, thread_data)
return json.dumps(
{
"status": "ready",
"path": requested_path,
"description": str(description or ""),
"append": append,
},
ensure_ascii=False,
separators=(",", ":"),
)
except SandboxError as e:
return f"Error: {e}"
except PermissionError:
return f"Error: Permission denied writing to file: {requested_path}"
except IsADirectoryError:
return f"Error: Path is a directory, not a file: {requested_path}"
except OSError as e:
return f"Error: Failed to prepare file '{requested_path}': {_sanitize_error(e, runtime)}"
except Exception as e:
return f"Error: Unexpected error preparing file: {_sanitize_error(e, runtime)}"
@tool("str_replace", parse_docstring=True)
@with_skill_view_scope
def str_replace_tool(
runtime: ToolRuntime[ContextT, ThreadState],
description: str,
@@ -1629,6 +1957,7 @@ def str_replace_tool(
content = content.replace(old_str, new_str)
else:
content = content.replace(old_str, new_str, 1)
content = maybe_fix_generated_html(path, content)
sandbox.write_file(path, content)
return "OK"
except SandboxError as e: