sync update from LAN repo (source dhk-1 @ c21a2a4a)
This commit is contained in:
1 parent
264e2200d6
commit
a1fa13a238
122 files changed
+8881
-623
No files matched your search
+10
-8
@@ -5,26 +5,28 @@ import pytest
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
_SKILL_VIEW_KEYS = ("skill_view_reuse_enabled", "skill_view_cache_enabled")
|
||||
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled", "tool_metrics_enabled")
|
||||
_DEFAULT_ON_KEYS = (*_SKILL_VIEW_KEYS, "tool_metrics_enabled", "skill_metrics_enabled")
|
||||
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled")
|
||||
|
||||
|
||||
def test_missing_runtime_performance_enables_skill_view_only():
|
||||
def test_missing_runtime_performance_enables_skill_view_and_skill_metrics():
|
||||
config = AppConfig(sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"})
|
||||
assert config.runtime_performance.model_dump() == {
|
||||
"skill_view_reuse_enabled": True,
|
||||
"skill_view_cache_enabled": True,
|
||||
"tool_metrics_enabled": False,
|
||||
"tool_metrics_enabled": True,
|
||||
"skill_metrics_enabled": True,
|
||||
"model_timing_nonblocking_enabled": False,
|
||||
"asyncio_thread_pool_enabled": False,
|
||||
"asyncio_thread_pool_size": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [*_SKILL_VIEW_KEYS, *_OPT_IN_KEYS])
|
||||
@pytest.mark.parametrize("key", [*_DEFAULT_ON_KEYS, *_OPT_IN_KEYS])
|
||||
def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
|
||||
# Skill-view switches default on, so independence is checked by turning
|
||||
# one of them off. The opt-in switches default off and are turned on.
|
||||
enabled = key not in _SKILL_VIEW_KEYS
|
||||
# Skill-view and audit metrics default on, so independence is checked by
|
||||
# turning one off. Opt-in switches default off and are turned on.
|
||||
enabled = key not in _DEFAULT_ON_KEYS
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
f"sandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\nruntime_performance:\n {key}: {'true' if enabled else 'false'}\n",
|
||||
@@ -32,7 +34,7 @@ def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
|
||||
)
|
||||
flags = AppConfig.from_file(str(config_path)).runtime_performance.model_dump()
|
||||
assert flags[key] is enabled
|
||||
for other in _SKILL_VIEW_KEYS:
|
||||
for other in _DEFAULT_ON_KEYS:
|
||||
if other != key:
|
||||
assert flags[other] is True
|
||||
for other in _OPT_IN_KEYS:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Regression coverage for model-independent skill usage attribution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.sandbox import tools
|
||||
from deerflow.skills import usage
|
||||
from deerflow.tools.builtins import skill_tools
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_recent_use_events():
|
||||
usage._recent_use_events.clear()
|
||||
yield
|
||||
usage._recent_use_events.clear()
|
||||
|
||||
|
||||
def test_bump_use_counts_a_skill_once_per_run_and_syncs_the_catalog(tmp_path: Path, monkeypatch):
|
||||
usage_path = tmp_path / ".usage.json"
|
||||
projected: list[tuple[str, int, object]] = []
|
||||
monkeypatch.setattr(usage, "_usage_path", lambda: usage_path)
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"_schedule_use_projection_increment",
|
||||
lambda name, use_count, used_at: projected.append((name, use_count, used_at)),
|
||||
)
|
||||
|
||||
assert usage.bump_use("page-generator", event_key="run-1") is True
|
||||
assert usage.bump_use("page-generator", event_key="run-1") is False
|
||||
assert usage.bump_use("page-generator", event_key="run-2") is True
|
||||
|
||||
entry = usage.get_entry("page-generator")
|
||||
assert entry["use_count"] == 2
|
||||
assert entry["last_used_at"]
|
||||
assert [(name, count) for name, count, _used_at in projected] == [
|
||||
("page-generator", 1),
|
||||
("page-generator", 2),
|
||||
]
|
||||
|
||||
|
||||
def test_skill_file_and_script_paths_record_public_and_custom_skills(monkeypatch):
|
||||
calls: list[tuple[str, str | None]] = []
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"bump_use",
|
||||
lambda name, *, event_key=None: calls.append((name, event_key)),
|
||||
)
|
||||
runtime = SimpleNamespace(context={"run_id": "run-weak-model"}, config={})
|
||||
|
||||
tools._record_skill_read("/mnt/skills/public/report-maker/SKILL.md", runtime)
|
||||
tools._record_skill_command(
|
||||
"python /mnt/skills/custom/data-cleaner/scripts/run.py --fast",
|
||||
runtime,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("report-maker", "run-weak-model"),
|
||||
("data-cleaner", "run-weak-model"),
|
||||
]
|
||||
|
||||
|
||||
def test_skill_view_counts_as_use_without_model_narration(monkeypatch):
|
||||
calls: list[tuple[str, str | None]] = []
|
||||
runtime = SimpleNamespace(context={"run_id": "run-skill-view"}, config={})
|
||||
monkeypatch.setattr(skill_tools, "skill_view_impl", lambda *_args, **_kwargs: "skill instructions")
|
||||
monkeypatch.setattr(
|
||||
usage,
|
||||
"bump_use",
|
||||
lambda name, *, event_key=None: calls.append((name, event_key)),
|
||||
)
|
||||
|
||||
result = skill_tools.skill_view_tool.func(name="report-maker", runtime=runtime)
|
||||
|
||||
assert result == "skill instructions"
|
||||
assert calls == [("report-maker", "run-skill-view")]
|
||||
|
||||
|
||||
def test_failed_skill_view_does_not_count_as_use(monkeypatch):
|
||||
calls: list[str] = []
|
||||
runtime = SimpleNamespace(context={"run_id": "run-missing"}, config={})
|
||||
monkeypatch.setattr(
|
||||
skill_tools,
|
||||
"skill_view_impl",
|
||||
lambda *_args, **_kwargs: "Skill 'missing' not found.",
|
||||
)
|
||||
monkeypatch.setattr(usage, "bump_use", lambda name, **_kwargs: calls.append(name))
|
||||
|
||||
result = skill_tools.skill_view_tool.func(name="missing", runtime=runtime)
|
||||
|
||||
assert result == "Skill 'missing' not found."
|
||||
assert calls == []
|
||||
+25
-1
@@ -288,7 +288,7 @@ def _tool_sandbox(tmp_path: Path, view_root: Path, monkeypatch: pytest.MonkeyPat
|
||||
monkeypatch.setattr(tools, "is_local_sandbox", lambda _runtime: True)
|
||||
monkeypatch.setattr(tools, "is_host_bash_allowed", lambda: True)
|
||||
monkeypatch.setattr(tools, "get_thread_data", lambda _runtime: {"workspace_path": str(tmp_path / "workspace"), "uploads_path": str(tmp_path / "uploads"), "outputs_path": str(tmp_path / "outputs")})
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda _path: None)
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda *_args: None)
|
||||
monkeypatch.setattr(tools, "_get_skills_container_path", lambda: "/mnt/skills")
|
||||
(tmp_path / "workspace").mkdir(exist_ok=True)
|
||||
assert view_root.is_dir()
|
||||
@@ -326,6 +326,30 @@ def test_bash_runs_the_updated_skill_script(cache_setup, monkeypatch, tmp_path):
|
||||
assert (view.root / "public" / "online" / "scripts" / "run.py").read_text(encoding="utf-8") == "print('two')\n"
|
||||
|
||||
|
||||
def test_bash_records_a_direct_skill_script_invocation(cache_setup, monkeypatch, tmp_path):
|
||||
root, _config, storage = cache_setup
|
||||
script = root / "public" / "online" / "scripts"
|
||||
script.mkdir()
|
||||
(script / "run.py").write_text("print('used')\n", encoding="utf-8")
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
recorded: list[tuple[str, object]] = []
|
||||
monkeypatch.setattr(
|
||||
tools,
|
||||
"_record_skill_command",
|
||||
lambda command, runtime: recorded.append((command, runtime)),
|
||||
)
|
||||
|
||||
result = tools.bash_tool.func(
|
||||
runtime=None,
|
||||
description="test",
|
||||
command="python /mnt/skills/public/online/scripts/run.py",
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert recorded == [("python /mnt/skills/public/online/scripts/run.py", None)]
|
||||
|
||||
|
||||
def test_bash_rejects_an_offline_skill(cache_setup, monkeypatch, tmp_path):
|
||||
root, config, storage = cache_setup
|
||||
_skill(root, "offline")
|
||||
|
||||
+18
@@ -50,3 +50,21 @@ def test_canonical_write_file_is_still_recorded(monkeypatch) -> None:
|
||||
middleware.wrap_tool_call(_request("write_file"), lambda _: _result("write_file"))
|
||||
|
||||
flush.assert_called_once()
|
||||
|
||||
|
||||
def test_call_metrics_switch_skips_new_metric_rows(monkeypatch) -> None:
|
||||
middleware = ToolMetricsMiddleware()
|
||||
flush = Mock()
|
||||
monkeypatch.setattr(middleware, "_flush", flush)
|
||||
monkeypatch.setattr(
|
||||
"deerflow.agents.middlewares.tool_metrics_middleware.get_call_metrics_settings",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
|
||||
result = middleware.wrap_tool_call(
|
||||
_request("write_file"),
|
||||
lambda _: _result("write_file"),
|
||||
)
|
||||
|
||||
assert result.name == "write_file"
|
||||
flush.assert_not_called()
|
||||
@@ -20,8 +20,10 @@ from deerflow.config.app_config import AppConfig
|
||||
BUILDERS = [build_lead_runtime_middlewares, build_subagent_runtime_middlewares]
|
||||
|
||||
|
||||
def _config(enabled=None):
|
||||
def _config(enabled=None, *, skill_enabled=None):
|
||||
options = {} if enabled is None else {"tool_metrics_enabled": enabled}
|
||||
if skill_enabled is not None:
|
||||
options["skill_metrics_enabled"] = skill_enabled
|
||||
return AppConfig(
|
||||
sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
runtime_performance=options,
|
||||
@@ -29,18 +31,29 @@ def _config(enabled=None):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
@pytest.mark.parametrize("enabled", [None, False])
|
||||
def test_paused_runtime_never_constructs_a_recorder(builder, enabled, monkeypatch):
|
||||
def test_explicitly_paused_runtime_never_constructs_a_recorder(builder, monkeypatch):
|
||||
constructor = Mock(side_effect=AssertionError("paused runtime constructed a DB recorder"))
|
||||
monkeypatch.setattr(metrics, "ToolMetricsMiddleware", constructor)
|
||||
|
||||
middlewares = builder(app_config=_config(enabled))
|
||||
middlewares = builder(app_config=_config(False, skill_enabled=False))
|
||||
|
||||
constructor.assert_not_called()
|
||||
assert any(isinstance(m, ToolErrorHandlingMiddleware) for m in middlewares)
|
||||
assert any(isinstance(m, LlmMetricsMiddleware) for m in middlewares)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
def test_default_runtime_keeps_a_full_tool_and_skill_recorder(builder):
|
||||
recorders = [
|
||||
middleware
|
||||
for middleware in builder(app_config=_config())
|
||||
if isinstance(middleware, metrics.ToolMetricsMiddleware)
|
||||
]
|
||||
|
||||
assert len(recorders) == 1
|
||||
assert recorders[0]._skill_only is False
|
||||
|
||||
|
||||
def _tool_wrappers(builder, enabled):
|
||||
# Exercise the actual error/audit wrappers selected by both runtime builders.
|
||||
return [m for m in builder(app_config=_config(enabled)) if isinstance(m, (ToolErrorHandlingMiddleware, metrics.ToolMetricsMiddleware))]
|
||||
@@ -77,6 +90,30 @@ def test_sync_tool_execution_and_error_conversion_survive_pause(builder, enabled
|
||||
assert flushed.call_args.args[0].status == result.status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
def test_skill_only_mode_records_a_path_based_skill_call(builder, monkeypatch):
|
||||
flushed = Mock()
|
||||
monkeypatch.setattr(metrics.ToolMetricsMiddleware, "_flush", lambda _, record: flushed(record))
|
||||
request = SimpleNamespace(
|
||||
tool_call={
|
||||
"id": "call-skill",
|
||||
"name": "bash",
|
||||
"args": {"command": "python /mnt/skills/public/report-maker/scripts/run.py"},
|
||||
}
|
||||
)
|
||||
expected = ToolMessage(content="ok", tool_call_id="call-skill", name="bash")
|
||||
|
||||
def handler(_request):
|
||||
return expected
|
||||
|
||||
for middleware in reversed(_tool_wrappers(builder, False)):
|
||||
handler = partial(middleware.wrap_tool_call, handler=handler)
|
||||
|
||||
assert handler(request) is expected
|
||||
assert flushed.call_count == 1
|
||||
assert flushed.call_args.args[0].skill_name == "report-maker"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("builder", BUILDERS)
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
|
||||
Reference in new issue
Block a user