sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
@@ -0,0 +1,69 @@
|
||||
"""Agent build timing must not move work out of its original task context."""
|
||||
|
||||
import logging
|
||||
from contextvars import ContextVar
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.runtime.runs import worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize("supports_app_config", [False, True])
|
||||
def test_build_preserves_config_and_contextvars(monkeypatch, caplog, supports_app_config):
|
||||
clock = iter([10.0, 10.6])
|
||||
monkeypatch.setattr(worker, "perf_counter", lambda: next(clock))
|
||||
state = ContextVar("test_build_context", default="before")
|
||||
config = {"configurable": {"test": True}}
|
||||
app_config = object()
|
||||
agent = object()
|
||||
|
||||
def modern_factory(*, config, app_config):
|
||||
assert config is expected_config
|
||||
assert app_config is expected_app_config
|
||||
state.set("after")
|
||||
return agent
|
||||
|
||||
def legacy_factory(*, config):
|
||||
assert config is expected_config
|
||||
state.set("after")
|
||||
return agent
|
||||
|
||||
expected_config, expected_app_config = config, app_config
|
||||
factory = modern_factory if supports_app_config else legacy_factory
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
assert worker._build_agent_with_timing(factory, config, app_config, "test-run") is agent
|
||||
assert state.get() == "after"
|
||||
assert "AGENT_BUILD_TIMING run_id=test-run outcome=ok elapsed_ms=600.00" in caplog.text
|
||||
assert caplog.records[-1].levelno == logging.WARNING
|
||||
|
||||
|
||||
def test_build_failure_preserves_exception_and_does_not_log_content(monkeypatch, caplog):
|
||||
clock = iter([10.0, 10.1])
|
||||
monkeypatch.setattr(worker, "perf_counter", lambda: next(clock))
|
||||
failure = ValueError("private prompt must not appear in timing logs")
|
||||
|
||||
def factory(*, config):
|
||||
raise failure
|
||||
|
||||
with caplog.at_level(logging.DEBUG), pytest.raises(ValueError) as captured:
|
||||
worker._build_agent_with_timing(factory, {}, None, "failed-run")
|
||||
assert captured.value is failure
|
||||
assert "outcome=error elapsed_ms=100.00" in caplog.text
|
||||
assert "private prompt" not in caplog.text
|
||||
assert caplog.records[-1].levelno == logging.DEBUG
|
||||
|
||||
|
||||
def test_build_timing_handler_failure_does_not_replace_factory_error(monkeypatch):
|
||||
failure = ValueError("factory failed")
|
||||
|
||||
def factory(*, config):
|
||||
raise failure
|
||||
|
||||
def broken_log(*args, **kwargs):
|
||||
raise RuntimeError("log handler failed")
|
||||
|
||||
monkeypatch.setattr(worker.logger, "debug", broken_log)
|
||||
monkeypatch.setattr(worker.logger, "warning", broken_log)
|
||||
with pytest.raises(ValueError) as captured:
|
||||
worker._build_agent_with_timing(factory, {}, None, "failed-run")
|
||||
assert captured.value is failure
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
"""Gateway executor configuration without importing the full application."""
|
||||
|
||||
import ast
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway import runtime_executor
|
||||
from deerflow.config.runtime_performance_config import RuntimePerformanceConfig
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_executor_environment(monkeypatch):
|
||||
monkeypatch.delenv("DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", raising=False)
|
||||
monkeypatch.delenv("DEERFLOW_IO_THREADS", raising=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_executor(monkeypatch):
|
||||
loop = Mock()
|
||||
factory = Mock()
|
||||
monkeypatch.setattr(runtime_executor, "asyncio", Mock(get_running_loop=Mock(return_value=loop)))
|
||||
monkeypatch.setattr(runtime_executor, "ThreadPoolExecutor", factory)
|
||||
return loop, factory
|
||||
|
||||
|
||||
def test_disabled_preserves_python_default_even_with_environment(monkeypatch, fake_executor, caplog):
|
||||
monkeypatch.setenv("DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", "invalid-unused-value")
|
||||
monkeypatch.setenv("DEERFLOW_IO_THREADS", "100")
|
||||
loop, factory = fake_executor
|
||||
with caplog.at_level(logging.INFO):
|
||||
asyncio.run(runtime_executor.configure_runtime_executor(RuntimePerformanceConfig()))
|
||||
factory.assert_not_called()
|
||||
loop.set_default_executor.assert_not_called()
|
||||
runtime_executor.asyncio.get_running_loop.assert_not_called()
|
||||
assert "enabled=false" in caplog.text
|
||||
assert "source=python_default" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("canonical", "alias", "config_size", "expected", "source"),
|
||||
[
|
||||
(None, None, 100, 100, "config.runtime_performance.asyncio_thread_pool_size"),
|
||||
(None, None, 24, 24, "config.runtime_performance.asyncio_thread_pool_size"),
|
||||
("", None, 24, 24, "config.runtime_performance.asyncio_thread_pool_size"),
|
||||
(" ", "100", 24, 100, "DEERFLOW_IO_THREADS"),
|
||||
(None, "", 24, 24, "config.runtime_performance.asyncio_thread_pool_size"),
|
||||
(None, "100", 24, 100, "DEERFLOW_IO_THREADS"),
|
||||
("100", None, 24, 100, "DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE"),
|
||||
("100", "64", 24, 100, "DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE"),
|
||||
("100", "invalid-unused-value", 24, 100, "DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE"),
|
||||
("1", None, 24, 1, "DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE"),
|
||||
(None, "1024", 24, 1024, "DEERFLOW_IO_THREADS"),
|
||||
],
|
||||
)
|
||||
def test_enabled_size_and_environment_priority(monkeypatch, fake_executor, caplog, canonical, alias, config_size, expected, source):
|
||||
if canonical is not None:
|
||||
monkeypatch.setenv("DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", canonical)
|
||||
if alias is not None:
|
||||
monkeypatch.setenv("DEERFLOW_IO_THREADS", alias)
|
||||
loop, factory = fake_executor
|
||||
performance = RuntimePerformanceConfig(asyncio_thread_pool_enabled=True, asyncio_thread_pool_size=config_size)
|
||||
with caplog.at_level(logging.INFO):
|
||||
asyncio.run(runtime_executor.configure_runtime_executor(performance))
|
||||
factory.assert_called_once_with(max_workers=expected, thread_name_prefix="gateway-sync")
|
||||
loop.set_default_executor.assert_called_once_with(factory.return_value)
|
||||
assert "enabled=true" in caplog.text
|
||||
assert f"max_workers={expected}" in caplog.text
|
||||
assert f"source={source}" in caplog.text
|
||||
assert "pid=" in caplog.text
|
||||
if canonical and canonical.strip() and alias and alias.strip():
|
||||
assert "DEERFLOW_IO_THREADS" in caplog.text
|
||||
assert "ignored" in caplog.text
|
||||
assert "invalid-unused-value" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variable", ["DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", "DEERFLOW_IO_THREADS"])
|
||||
@pytest.mark.parametrize("invalid", ["0", "-1", "1025", "1.0", "true", "1_00", "secret-invalid-value", "1\n00"])
|
||||
def test_invalid_selected_environment_fails_before_executor_install(monkeypatch, fake_executor, variable, invalid):
|
||||
monkeypatch.setenv(variable, invalid)
|
||||
loop, factory = fake_executor
|
||||
performance = RuntimePerformanceConfig(asyncio_thread_pool_enabled=True)
|
||||
with pytest.raises(ValueError, match=variable) as failure:
|
||||
asyncio.run(runtime_executor.configure_runtime_executor(performance))
|
||||
assert "secret-invalid-value" not in str(failure.value)
|
||||
factory.assert_not_called()
|
||||
loop.set_default_executor.assert_not_called()
|
||||
|
||||
|
||||
def test_invalid_canonical_does_not_silently_use_valid_alias(monkeypatch, fake_executor):
|
||||
monkeypatch.setenv("DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", "invalid")
|
||||
monkeypatch.setenv("DEERFLOW_IO_THREADS", "100")
|
||||
performance = RuntimePerformanceConfig(asyncio_thread_pool_enabled=True)
|
||||
with pytest.raises(ValueError, match="DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE"):
|
||||
asyncio.run(runtime_executor.configure_runtime_executor(performance))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker_count", [24, 100])
|
||||
def test_configured_executor_runs_requested_blocking_tools_and_runner_cleans_up(monkeypatch, worker_count):
|
||||
instances = []
|
||||
barrier = threading.Barrier(worker_count, timeout=20)
|
||||
|
||||
def factory(**kwargs):
|
||||
executor = ThreadPoolExecutor(**kwargs)
|
||||
instances.append(executor)
|
||||
return executor
|
||||
|
||||
def blocking_tool():
|
||||
# All requested workers must enter together: a pool still limited to
|
||||
# the YAML fallback of 12 cannot pass, including the 100-worker case.
|
||||
barrier.wait()
|
||||
return threading.current_thread().name
|
||||
|
||||
async def run_tools():
|
||||
await runtime_executor.configure_runtime_executor(RuntimePerformanceConfig(asyncio_thread_pool_enabled=True, asyncio_thread_pool_size=12))
|
||||
return await asyncio.gather(*(asyncio.to_thread(blocking_tool) for _ in range(worker_count)))
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_ASYNCIO_THREAD_POOL_SIZE", str(worker_count))
|
||||
monkeypatch.setattr(runtime_executor, "ThreadPoolExecutor", factory)
|
||||
names = asyncio.run(run_tools())
|
||||
assert len(set(names)) == worker_count
|
||||
assert all(name.startswith("gateway-sync") for name in names)
|
||||
assert len(instances) == 1
|
||||
assert not any(thread.is_alive() for thread in instances[0]._threads)
|
||||
|
||||
|
||||
def test_gateway_configures_executor_after_config_before_langgraph_startup():
|
||||
app_source = Path(runtime_executor.__file__).with_name("app.py").read_text(encoding="utf-8")
|
||||
tree = ast.parse(app_source)
|
||||
lifespan = next(node for node in tree.body if isinstance(node, ast.AsyncFunctionDef) and node.name == "lifespan")
|
||||
calls = {node.func.id: node for node in ast.walk(lifespan) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in {"get_app_config", "configure_runtime_executor", "langgraph_runtime"}}
|
||||
assert calls["get_app_config"].lineno < calls["configure_runtime_executor"].lineno < calls["langgraph_runtime"].lineno
|
||||
assert any(isinstance(node, ast.Await) and node.value is calls["configure_runtime_executor"] for node in ast.walk(lifespan))
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from deerflow.runtime.journal import RunJournal
|
||||
from deerflow.runtime.serialization import serialize_channel_values, serialize_messages_tuple
|
||||
from deerflow.utils.tool_visibility import project_outward_messages
|
||||
|
||||
|
||||
def _call(call_id: str, name: str, **args):
|
||||
return {"id": call_id, "name": name, "args": args, "type": "tool_call"}
|
||||
|
||||
|
||||
def test_pair_aware_projection_hides_handshake_and_preserves_canonical_reasoning() -> None:
|
||||
messages = [
|
||||
{"type": "human", "id": "h1", "content": "写报告"},
|
||||
{
|
||||
"type": "ai",
|
||||
"id": "a-begin",
|
||||
"content": "好的,我来写。",
|
||||
"additional_kwargs": {"reasoning_content": "准备目标路径"},
|
||||
"tool_calls": [_call("begin-1", "begin_write_file", path="/tmp/report.md")],
|
||||
},
|
||||
{
|
||||
"type": "tool",
|
||||
"id": "t-begin",
|
||||
"content": '{"status":"ready"}',
|
||||
"tool_call_id": "begin-1",
|
||||
},
|
||||
{
|
||||
"type": "ai",
|
||||
"id": "a-write",
|
||||
"content": "",
|
||||
"additional_kwargs": {"reasoning_content": "组织完整正文"},
|
||||
"tool_calls": [
|
||||
_call(
|
||||
"write-1",
|
||||
"write_file",
|
||||
path="/tmp/report.md",
|
||||
content="# 报告",
|
||||
)
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "tool",
|
||||
"id": "t-write",
|
||||
"name": "write_file",
|
||||
"content": "OK",
|
||||
"tool_call_id": "write-1",
|
||||
},
|
||||
]
|
||||
|
||||
projected = project_outward_messages(messages)
|
||||
|
||||
assert [message["id"] for message in projected] == ["h1", "a-write", "t-write"]
|
||||
assert projected[1]["additional_kwargs"]["reasoning_content"] == "组织完整正文"
|
||||
assert projected[1]["tool_calls"][0]["args"]["content"] == "# 报告"
|
||||
|
||||
|
||||
def test_projection_does_not_hide_later_visible_tool_when_provider_reuses_an_id() -> None:
|
||||
projected = project_outward_messages(
|
||||
[
|
||||
{
|
||||
"type": "ai",
|
||||
"tool_calls": [_call("reused-1", "begin_write_file", path="/tmp/a.md")],
|
||||
},
|
||||
{
|
||||
"type": "tool",
|
||||
"tool_call_id": "reused-1",
|
||||
"content": '{"status":"ready"}',
|
||||
},
|
||||
{
|
||||
"type": "ai",
|
||||
"tool_calls": [_call("reused-1", "read_file", path="/tmp/a.md")],
|
||||
},
|
||||
{
|
||||
"type": "tool",
|
||||
"tool_call_id": "reused-1",
|
||||
"name": "read_file",
|
||||
"content": "visible result",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert [message.get("name") for message in projected] == [None, "read_file"]
|
||||
assert projected[0]["tool_calls"][0]["name"] == "read_file"
|
||||
|
||||
|
||||
def test_projection_keeps_visible_calls_and_reasoning_from_a_mixed_message() -> None:
|
||||
projected = project_outward_messages(
|
||||
[
|
||||
{
|
||||
"type": "ai",
|
||||
"content": "visible preface",
|
||||
"additional_kwargs": {"reasoning_content": "visible reasoning"},
|
||||
"tool_calls": [
|
||||
_call("begin-1", "begin_write_file", path="/tmp/a.md"),
|
||||
_call("read-1", "read_file", path="/tmp/source.md"),
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert projected[0]["content"] == "visible preface"
|
||||
assert projected[0]["additional_kwargs"]["reasoning_content"] == "visible reasoning"
|
||||
assert [call["name"] for call in projected[0]["tool_calls"]] == ["read_file"]
|
||||
|
||||
|
||||
def test_runtime_serialization_projects_values_and_single_internal_frames() -> None:
|
||||
begin = AIMessage(
|
||||
id="a-begin",
|
||||
content="selection preface",
|
||||
tool_calls=[_call("begin-1", "begin_write_file", path="/tmp/report.md")],
|
||||
)
|
||||
receipt = ToolMessage(
|
||||
id="t-begin",
|
||||
content='{"status":"ready"}',
|
||||
name="begin_write_file",
|
||||
tool_call_id="begin-1",
|
||||
)
|
||||
canonical = AIMessage(
|
||||
id="a-write",
|
||||
content="",
|
||||
additional_kwargs={"reasoning_content": "正文推理"},
|
||||
tool_calls=[
|
||||
_call(
|
||||
"write-1",
|
||||
"write_file",
|
||||
path="/tmp/report.md",
|
||||
content="# 报告",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
values = serialize_channel_values(
|
||||
{"messages": [HumanMessage(content="写报告"), begin, receipt, canonical]}
|
||||
)
|
||||
|
||||
assert [message.get("id") for message in values["messages"]] == [None, "a-write"]
|
||||
assert values["messages"][1]["additional_kwargs"]["reasoning_content"] == "正文推理"
|
||||
assert serialize_messages_tuple((begin, {"langgraph_node": "model"})) is None
|
||||
assert serialize_messages_tuple((receipt, {"langgraph_node": "tools"})) is None
|
||||
assert serialize_messages_tuple((canonical, {}))[0]["tool_calls"][0]["name"] == "write_file"
|
||||
|
||||
|
||||
def test_messages_stream_uses_call_id_to_hide_nameless_internal_result() -> None:
|
||||
call_ids: set[str] = set()
|
||||
begin = AIMessage(
|
||||
content="",
|
||||
tool_calls=[_call("begin-1", "begin_write_file", path="/tmp/report.md")],
|
||||
)
|
||||
nameless_receipt = ToolMessage(
|
||||
content='{"status":"ready"}',
|
||||
tool_call_id="begin-1",
|
||||
)
|
||||
|
||||
assert (
|
||||
serialize_messages_tuple(
|
||||
(begin, {}),
|
||||
known_internal_tool_call_ids=call_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert call_ids == {"begin-1"}
|
||||
assert (
|
||||
serialize_messages_tuple(
|
||||
(nameless_receipt, {}),
|
||||
known_internal_tool_call_ids=call_ids,
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert call_ids == set()
|
||||
|
||||
|
||||
def test_run_journal_never_persists_internal_call_or_paired_result() -> None:
|
||||
journal = RunJournal(
|
||||
run_id="run-1",
|
||||
thread_id="thread-1",
|
||||
event_store=object(),
|
||||
flush_threshold=100,
|
||||
)
|
||||
begin = AIMessage(
|
||||
content="selection preface",
|
||||
tool_calls=[_call("begin-1", "begin_write_file", path="/tmp/report.md")],
|
||||
)
|
||||
response = SimpleNamespace(
|
||||
generations=[[SimpleNamespace(message=begin)]],
|
||||
)
|
||||
journal.on_llm_end(response, run_id=uuid4())
|
||||
journal.on_tool_end(
|
||||
ToolMessage(content='{"status":"ready"}', tool_call_id="begin-1"),
|
||||
run_id=uuid4(),
|
||||
)
|
||||
|
||||
canonical = AIMessage(
|
||||
content="",
|
||||
additional_kwargs={"reasoning_content": "正文推理"},
|
||||
tool_calls=[_call("write-1", "write_file", path="/tmp/report.md", content="# 报告")],
|
||||
)
|
||||
journal.on_llm_end(
|
||||
SimpleNamespace(generations=[[SimpleNamespace(message=canonical)]]),
|
||||
run_id=uuid4(),
|
||||
)
|
||||
|
||||
message_events = [event for event in journal._buffer if event["category"] == "message"]
|
||||
assert len(message_events) == 1
|
||||
assert message_events[0]["content"]["tool_calls"][0]["name"] == "write_file"
|
||||
assert message_events[0]["content"]["additional_kwargs"]["reasoning_content"] == "正文推理"
|
||||
@@ -0,0 +1,33 @@
|
||||
from deerflow.integrations.weknora.local_index.ranking import (
|
||||
title_boosted_rrf_score,
|
||||
title_match_strength,
|
||||
)
|
||||
|
||||
|
||||
def test_exact_chinese_title_is_strongest_match() -> None:
|
||||
exact = title_match_strength("海湖庄园", "海湖庄园")
|
||||
broad = title_match_strength("海湖庄园", "特朗普")
|
||||
assert exact == 1.0
|
||||
assert broad == 0.0
|
||||
|
||||
|
||||
def test_question_wrappers_do_not_hide_title_phrase() -> None:
|
||||
assert title_match_strength("请介绍一下海湖庄园的历史", "海湖庄园") == 1.0
|
||||
|
||||
|
||||
def test_title_inside_longer_natural_language_question_is_retained() -> None:
|
||||
assert title_match_strength("请查找海湖庄园最新消息", "海湖庄园") > 0.7
|
||||
|
||||
|
||||
def test_possessive_character_inside_real_title_is_not_deleted() -> None:
|
||||
assert title_match_strength("党的二十大", "党的二十大") == 1.0
|
||||
|
||||
|
||||
def test_alias_match_is_discounted_but_still_useful() -> None:
|
||||
assert 0 < title_match_strength("Mar-a-Lago", "美国总统官邸", ["Mar-a-Lago"]) < 1
|
||||
|
||||
|
||||
def test_rrf_title_boost_is_weighted_and_can_be_disabled() -> None:
|
||||
base = 1 / 62
|
||||
assert title_boosted_rrf_score(base, 1.0, weight=0.25) == base * 1.25
|
||||
assert title_boosted_rrf_score(base, 1.0, enabled=False, weight=1.0) == base
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""A failed final file commit must preserve the previous artifact."""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sandbox(tmp_path):
|
||||
return LocalSandbox("atomic-write", [PathMapping("/artifacts", str(tmp_path))])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exists", [False, True])
|
||||
@pytest.mark.parametrize("failure", ["encoding", "fsync", "replace"])
|
||||
def test_failed_overwrite_preserves_target_and_cleans_staging(sandbox, tmp_path, monkeypatch, exists, failure):
|
||||
target = tmp_path / "report.md"
|
||||
if exists:
|
||||
target.write_text("previous report", encoding="utf-8")
|
||||
|
||||
def fail(*args):
|
||||
raise OSError(errno.ENOSPC, "Simulated disk failure")
|
||||
|
||||
content = "new report"
|
||||
if failure == "encoding":
|
||||
content += chr(0xD800)
|
||||
else:
|
||||
monkeypatch.setattr(os, failure, fail)
|
||||
with pytest.raises((OSError, UnicodeError)):
|
||||
sandbox.write_file("/artifacts/report.md", content)
|
||||
if exists:
|
||||
assert target.read_text(encoding="utf-8") == "previous report"
|
||||
else:
|
||||
assert not target.exists()
|
||||
assert not list(tmp_path.glob(".deerflow-write-*"))
|
||||
|
||||
|
||||
def test_successful_overwrite_and_append(sandbox, tmp_path):
|
||||
target = tmp_path / "report.md"
|
||||
sandbox.write_file("/artifacts/report.md", "first")
|
||||
sandbox.write_file("/artifacts/report.md", "replacement")
|
||||
sandbox.write_file("/artifacts/report.md", "\n追加", append=True)
|
||||
assert target.read_text(encoding="utf-8") == "replacement\n追加"
|
||||
assert not list(tmp_path.glob(".deerflow-write-*"))
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits")
|
||||
def test_overwrite_preserves_mode(sandbox, tmp_path):
|
||||
target = tmp_path / "report.md"
|
||||
target.write_text("old", encoding="utf-8")
|
||||
target.chmod(0o640)
|
||||
sandbox.write_file("/artifacts/report.md", "new")
|
||||
assert stat.S_IMODE(target.stat().st_mode) == 0o640
|
||||
|
||||
|
||||
def test_directory_target_is_not_replaced(sandbox, tmp_path):
|
||||
target = tmp_path / "report.md"
|
||||
target.mkdir()
|
||||
with pytest.raises(IsADirectoryError):
|
||||
sandbox.write_file("/artifacts/report.md", "new")
|
||||
assert target.is_dir()
|
||||
assert not list(tmp_path.glob(".deerflow-write-*"))
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
|
||||
|
||||
def _state(name: str) -> dict:
|
||||
return {
|
||||
"messages": [
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": f"call-{name}",
|
||||
"name": name,
|
||||
"args": {"path": "/mnt/user-data/outputs/report.md"},
|
||||
"type": "tool_call",
|
||||
}
|
||||
],
|
||||
)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_internal_begin_write_file_does_not_affect_loop_tracking() -> None:
|
||||
middleware = LoopDetectionMiddleware(
|
||||
warn_threshold=2,
|
||||
hard_limit=3,
|
||||
tool_freq_warn=2,
|
||||
tool_freq_hard_limit=3,
|
||||
)
|
||||
runtime = SimpleNamespace(context={"thread_id": "thread-1"})
|
||||
|
||||
for _ in range(10):
|
||||
assert middleware._track_and_check(_state("begin_write_file"), runtime) == (None, False)
|
||||
|
||||
assert "thread-1" not in middleware._history
|
||||
assert "thread-1" not in middleware._tool_freq
|
||||
|
||||
assert middleware._track_and_check(_state("write_file"), runtime) == (None, False)
|
||||
warning, hard_stop = middleware._track_and_check(_state("write_file"), runtime)
|
||||
assert warning is not None
|
||||
assert hard_stop is False
|
||||
@@ -0,0 +1,49 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow.config import memory_config as memory_config_module
|
||||
from deerflow.config.memory_config import (
|
||||
MemoryConfig,
|
||||
delete_recall_budget_cache,
|
||||
get_effective_memory_config,
|
||||
get_memory_config,
|
||||
set_memory_config,
|
||||
set_recall_budget_cache,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_disable_cannot_be_reactivated_by_config_or_user_override(monkeypatch):
|
||||
user_id = "memory-disabled-test-user"
|
||||
original = get_memory_config()
|
||||
try:
|
||||
monkeypatch.setattr(
|
||||
memory_config_module,
|
||||
"get_memory_runtime_settings",
|
||||
lambda: SimpleNamespace(enabled=False),
|
||||
)
|
||||
set_memory_config(MemoryConfig(enabled=True))
|
||||
set_recall_budget_cache(user_id, "high")
|
||||
|
||||
effective = get_effective_memory_config(user_id)
|
||||
|
||||
assert effective.enabled is False
|
||||
finally:
|
||||
delete_recall_budget_cache(user_id)
|
||||
set_memory_config(original)
|
||||
|
||||
|
||||
def test_admin_runtime_enable_activates_memory_even_when_yaml_default_is_off(monkeypatch):
|
||||
original = get_memory_config()
|
||||
try:
|
||||
monkeypatch.setattr(
|
||||
memory_config_module,
|
||||
"get_memory_runtime_settings",
|
||||
lambda: SimpleNamespace(enabled=True),
|
||||
)
|
||||
set_memory_config(MemoryConfig(enabled=False, version="v1"))
|
||||
|
||||
effective = get_effective_memory_config("memory-enabled-test-user")
|
||||
|
||||
assert effective.enabled is True
|
||||
assert effective.version == "v1"
|
||||
finally:
|
||||
set_memory_config(original)
|
||||
@@ -2,7 +2,7 @@ import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
|
||||
|
||||
from deerflow.agents.middlewares.writing_setup_round_cap_middleware import WritingSetupRoundCapMiddleware
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages, normalize_langchain_messages
|
||||
from deerflow.models.message_ordering import normalize_chat_payload_messages, normalize_langchain_messages, normalize_model_request_messages
|
||||
from deerflow.models.patched_openai import PatchedChatOpenAI
|
||||
|
||||
|
||||
@@ -49,6 +49,37 @@ def test_normalize_langchain_messages_moves_late_systems_before_history():
|
||||
assert normalized[1].content == "user 0"
|
||||
|
||||
|
||||
def test_normalize_chat_payload_messages_merges_adjacent_leading_systems():
|
||||
payload = {
|
||||
"messages": [
|
||||
{"role": "system", "content": "agent prompt"},
|
||||
{"role": "system", "content": "compaction summary"},
|
||||
{"role": "user", "content": "continue"},
|
||||
]
|
||||
}
|
||||
|
||||
normalize_chat_payload_messages(payload)
|
||||
|
||||
assert [message["role"] for message in payload["messages"]] == ["system", "user"]
|
||||
assert payload["messages"][0]["content"] == "agent prompt\n\n---\n\ncompaction summary"
|
||||
|
||||
|
||||
def test_normalize_model_request_messages_folds_compaction_summary_into_system_slot():
|
||||
system_message, messages = normalize_model_request_messages(
|
||||
SystemMessage(content="agent prompt"),
|
||||
[
|
||||
SystemMessage(content="compaction summary"),
|
||||
HumanMessage(content="continue"),
|
||||
],
|
||||
)
|
||||
|
||||
assert system_message is not None
|
||||
assert system_message.content == "agent prompt\n\n---\n\ncompaction summary"
|
||||
assert len(messages) == 1
|
||||
assert isinstance(messages[0], HumanMessage)
|
||||
assert all(not isinstance(message, SystemMessage) for message in messages)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("turns", [12, 25, 50])
|
||||
def test_patched_openai_payload_has_only_leading_system_after_many_turns(turns):
|
||||
model = PatchedChatOpenAI(
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config import runtime_performance_config
|
||||
from deerflow.models.timing_callback import ModelTimingCallback
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def synchronous_timing_logs(monkeypatch):
|
||||
# These assertions inspect logs immediately. Keep the legacy path isolated
|
||||
# from a developer's opt-in config; the queued path has its own regression
|
||||
# suite in test_model_timing_nonblocking.py.
|
||||
monkeypatch.setattr(
|
||||
runtime_performance_config,
|
||||
"get_runtime_performance_config",
|
||||
lambda: SimpleNamespace(model_timing_nonblocking_enabled=False),
|
||||
)
|
||||
|
||||
|
||||
def test_model_timing_callback_logs_connection_and_completion(caplog):
|
||||
callback = ModelTimingCallback(model_name="qwen", base_url="http://model/v1")
|
||||
run_id = uuid4()
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.timing_callback"):
|
||||
callback.on_chat_model_start({}, [[object()]], run_id=run_id)
|
||||
callback.on_llm_new_token("", run_id=run_id, chunk=object())
|
||||
callback.on_llm_end(object(), run_id=run_id)
|
||||
|
||||
messages = [record.getMessage() for record in caplog.records]
|
||||
assert any("LLM_CONNECT_START" in message for message in messages)
|
||||
assert any("LLM_CONNECT_OK" in message and "first_chunk_ms=" in message for message in messages)
|
||||
assert any("LLM_REQUEST_DONE" in message and "total_ms=" in message for message in messages)
|
||||
|
||||
|
||||
def test_model_timing_callback_logs_failure(caplog):
|
||||
callback = ModelTimingCallback(
|
||||
model_name="qwen",
|
||||
base_url="https://user:API_KEY_SHOULD_NOT_LOG@model.example:8443/v1?token=secret",
|
||||
)
|
||||
run_id = uuid4()
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.timing_callback"):
|
||||
callback.on_chat_model_start({}, [[object()]], run_id=run_id)
|
||||
callback.on_llm_error(TimeoutError("PROMPT_OR_TOOL_BODY_SHOULD_NOT_LOG"), run_id=run_id)
|
||||
|
||||
message = caplog.records[-1].getMessage()
|
||||
assert "LLM_REQUEST_FAILED" in message
|
||||
assert "TimeoutError" in message
|
||||
assert "base_url=https://model.example:8443" in message
|
||||
assert "API_KEY_SHOULD_NOT_LOG" not in message
|
||||
assert "PROMPT_OR_TOOL_BODY_SHOULD_NOT_LOG" not in message
|
||||
|
||||
|
||||
def test_model_timing_callback_logs_cancellation_as_info(caplog):
|
||||
callback = ModelTimingCallback(model_name="qwen", base_url="http://model/v1")
|
||||
run_id = uuid4()
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.timing_callback"):
|
||||
callback.on_chat_model_start({}, [[object()]], run_id=run_id)
|
||||
callback.on_llm_error(asyncio.CancelledError(), run_id=run_id)
|
||||
|
||||
cancellation_records = [record for record in caplog.records if "LLM_REQUEST_CANCELLED" in record.getMessage()]
|
||||
assert len(cancellation_records) == 1
|
||||
assert cancellation_records[0].levelno == logging.INFO
|
||||
assert "status=cancelled" in cancellation_records[0].getMessage()
|
||||
assert not any("LLM_REQUEST_FAILED" in record.getMessage() for record in caplog.records)
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
"""Opt-in model timing remains independent of busy tools and slow log sinks."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.language_models.chat_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
from langchain_core.outputs import ChatGenerationChunk
|
||||
|
||||
from deerflow.config import runtime_performance_config
|
||||
from deerflow.models import timing_callback
|
||||
from deerflow.models.timing_callback import ModelTimingCallback, _TimingLogDispatcher
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def timing_logs(monkeypatch, caplog):
|
||||
monkeypatch.setattr(runtime_performance_config, "get_runtime_performance_config", lambda: SimpleNamespace(model_timing_nonblocking_enabled=True))
|
||||
dispatcher = _TimingLogDispatcher()
|
||||
monkeypatch.setattr(timing_callback, "_timing_logs", dispatcher)
|
||||
caplog.set_level(logging.INFO, logger=timing_callback.logger.name)
|
||||
yield dispatcher
|
||||
dispatcher.close(timeout=2)
|
||||
|
||||
|
||||
class _NativeAsyncModel(BaseChatModel):
|
||||
@property
|
||||
def _llm_type(self):
|
||||
return "timing-callback-test"
|
||||
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
raise AssertionError("The model must use native async streaming")
|
||||
|
||||
async def _astream(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
yield ChatGenerationChunk(message=AIMessageChunk(content="first"))
|
||||
yield ChatGenerationChunk(message=AIMessageChunk(content="second"))
|
||||
|
||||
|
||||
class _SlowHandler(logging.Handler):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.entered = threading.Event()
|
||||
self.release_event = threading.Event()
|
||||
self.worker_threads = []
|
||||
|
||||
def emit(self, record):
|
||||
self.worker_threads.append(threading.current_thread().name)
|
||||
self.entered.set()
|
||||
self.release_event.wait(timeout=5)
|
||||
|
||||
|
||||
async def _wait_until_set(event):
|
||||
while not event.is_set():
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
|
||||
def test_default_off_keeps_synchronous_logging_and_executor_dispatch(monkeypatch, caplog):
|
||||
config = runtime_performance_config.RuntimePerformanceConfig()
|
||||
monkeypatch.setattr(runtime_performance_config, "get_runtime_performance_config", lambda: config)
|
||||
caplog.set_level(logging.INFO, logger=timing_callback.logger.name)
|
||||
callback = ModelTimingCallback(model_name="test", base_url=None)
|
||||
assert callback.run_inline is False
|
||||
|
||||
def reject_queue(*args):
|
||||
raise AssertionError("Default-off callbacks must not use the background log queue")
|
||||
|
||||
monkeypatch.setattr(timing_callback._timing_logs, "emit", reject_queue)
|
||||
run_id = uuid4()
|
||||
callback.on_chat_model_start({}, [[]], run_id=run_id)
|
||||
assert "LLM_CONNECT_START" in caplog.text
|
||||
callback.on_llm_new_token("token", run_id=run_id)
|
||||
assert "LLM_CONNECT_OK" in caplog.text
|
||||
callback.on_llm_end(None, run_id=run_id)
|
||||
assert "LLM_REQUEST_DONE" in caplog.text
|
||||
|
||||
|
||||
def test_stream_progresses_with_saturated_tool_pool_and_blocked_log_handler(timing_logs, caplog):
|
||||
"""Exercise LangChain dispatch, not just direct callback method calls."""
|
||||
sink = _SlowHandler()
|
||||
timing_callback.logger.addHandler(sink)
|
||||
tool_started = threading.Event()
|
||||
release_tool = threading.Event()
|
||||
|
||||
def long_tool():
|
||||
tool_started.set()
|
||||
release_tool.wait(timeout=5)
|
||||
|
||||
async def scenario():
|
||||
loop = asyncio.get_running_loop()
|
||||
# A fresh asyncio.run loop owns this executor and cleans it up.
|
||||
loop.set_default_executor(ThreadPoolExecutor(max_workers=1, thread_name_prefix="busy-tool"))
|
||||
blocked_tool = loop.run_in_executor(None, long_tool)
|
||||
try:
|
||||
await asyncio.wait_for(_wait_until_set(tool_started), timeout=1)
|
||||
callback = ModelTimingCallback(model_name="test", base_url="https://models.invalid/v1")
|
||||
model = _NativeAsyncModel(callbacks=[callback])
|
||||
|
||||
async def consume():
|
||||
return [chunk.content async for chunk in model.astream("private prompt")]
|
||||
|
||||
chunks = await asyncio.wait_for(consume(), timeout=1)
|
||||
await asyncio.wait_for(_wait_until_set(sink.entered), timeout=1)
|
||||
assert "".join(chunks) == "firstsecond"
|
||||
assert not blocked_tool.done()
|
||||
assert not sink.release_event.is_set()
|
||||
assert callback._calls == {}
|
||||
finally:
|
||||
release_tool.set()
|
||||
sink.release_event.set()
|
||||
await blocked_tool
|
||||
|
||||
try:
|
||||
asyncio.run(scenario())
|
||||
timing_logs.close(timeout=2)
|
||||
finally:
|
||||
release_tool.set()
|
||||
sink.release_event.set()
|
||||
timing_callback.logger.removeHandler(sink)
|
||||
assert sink.worker_threads
|
||||
assert set(sink.worker_threads) == {"deerflow-model-timing-log"}
|
||||
assert "LLM_CONNECT_START" in caplog.text
|
||||
assert "LLM_CONNECT_OK" in caplog.text
|
||||
assert "LLM_REQUEST_DONE" in caplog.text
|
||||
assert "private prompt" not in caplog.text
|
||||
|
||||
|
||||
def test_sync_callback_preserves_sanitized_lifecycle_and_only_logs_first_token(timing_logs, caplog):
|
||||
callback = ModelTimingCallback(model_name="test", base_url="https://user:secret@models.invalid:443/private?key=secret")
|
||||
run_id = uuid4()
|
||||
callback.on_chat_model_start({}, [["private input"]], run_id=run_id)
|
||||
callback.on_llm_new_token("", run_id=run_id)
|
||||
for _ in range(100):
|
||||
callback.on_llm_new_token("private output", run_id=run_id)
|
||||
callback.on_llm_end(None, run_id=run_id)
|
||||
timing_logs.close(timeout=2)
|
||||
|
||||
records = [record for record in caplog.records if record.name == timing_callback.logger.name]
|
||||
assert len(records) == 3
|
||||
assert "base_url=https://models.invalid:443" in caplog.text
|
||||
assert "first_chunk_ms=" in caplog.text
|
||||
assert f"run_id={run_id}" in caplog.text
|
||||
for secret in ("private", "secret", "user:"):
|
||||
assert secret not in caplog.text
|
||||
assert callback._calls == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected", "level"),
|
||||
[
|
||||
(RuntimeError("private upstream payload"), "LLM_REQUEST_FAILED", logging.ERROR),
|
||||
(asyncio.CancelledError("private cancellation"), "LLM_REQUEST_CANCELLED", logging.INFO),
|
||||
(GeneratorExit("private closure"), "LLM_REQUEST_CANCELLED", logging.INFO),
|
||||
],
|
||||
)
|
||||
def test_failure_and_cancellation_are_content_free(timing_logs, caplog, error, expected, level):
|
||||
callback = ModelTimingCallback(model_name="test", base_url=None)
|
||||
run_id = uuid4()
|
||||
callback.on_chat_model_start({}, [[]], run_id=run_id)
|
||||
callback.on_llm_error(error, run_id=run_id)
|
||||
timing_logs.close(timeout=2)
|
||||
|
||||
terminal = [record for record in caplog.records if expected in record.getMessage()]
|
||||
assert len(terminal) == 1
|
||||
assert terminal[0].levelno == level
|
||||
assert "private" not in caplog.text
|
||||
assert callback._calls == {}
|
||||
|
||||
|
||||
def test_full_log_queue_drops_without_blocking_and_reports_recovery(monkeypatch, caplog):
|
||||
dispatcher = _TimingLogDispatcher(capacity=1)
|
||||
monkeypatch.setattr(timing_callback, "_timing_logs", dispatcher)
|
||||
caplog.set_level(logging.INFO, logger=timing_callback.logger.name)
|
||||
sink = _SlowHandler()
|
||||
timing_callback.logger.addHandler(sink)
|
||||
try:
|
||||
dispatcher.emit(logging.INFO, "first")
|
||||
assert sink.entered.wait(timeout=1)
|
||||
dispatcher.emit(logging.INFO, "queued")
|
||||
dispatcher.emit(logging.INFO, "dropped")
|
||||
assert dispatcher._queue.qsize() == 1
|
||||
assert dispatcher._dropped == 1
|
||||
sink.release_event.set()
|
||||
dispatcher.close(timeout=2)
|
||||
finally:
|
||||
sink.release_event.set()
|
||||
dispatcher.close(timeout=2)
|
||||
timing_callback.logger.removeHandler(sink)
|
||||
assert "LLM_TIMING_LOG_DROPPED count=1 reason=queue_full" in caplog.text
|
||||
assert "queued" in caplog.text
|
||||
assert not any(record.getMessage() == "dropped" for record in caplog.records)
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""Regression contract for the production Gateway worker count.
|
||||
|
||||
The live run registry and memory stream bridge are process-local. Keep the
|
||||
offline production launchers single-worker unless those components gain a
|
||||
shared implementation.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _find_repo_root() -> Path:
|
||||
for parent in Path(__file__).resolve().parents:
|
||||
if (parent / "scripts" / "start_backend-online.sh").is_file():
|
||||
return parent
|
||||
raise RuntimeError("Could not find repository root from launcher test")
|
||||
|
||||
|
||||
REPO_ROOT = _find_repo_root()
|
||||
LAUNCHERS = (
|
||||
REPO_ROOT / "scripts" / "start_backend-online.sh",
|
||||
REPO_ROOT / "offline-backend-20260512" / "start_backend.sh",
|
||||
)
|
||||
|
||||
|
||||
def _launcher_text(launcher: Path) -> str:
|
||||
return launcher.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _working_bash() -> str:
|
||||
candidates: list[str | None] = []
|
||||
if os.name == "nt":
|
||||
program_files = os.environ.get("ProgramFiles", r"C:\Program Files")
|
||||
candidates.append(str(Path(program_files) / "Git" / "bin" / "bash.exe"))
|
||||
candidates.append(shutil.which("bash"))
|
||||
for candidate in candidates:
|
||||
if not candidate or not Path(candidate).is_file():
|
||||
continue
|
||||
try:
|
||||
check = subprocess.run(
|
||||
[candidate, "--version"],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
continue
|
||||
if check.returncode == 0:
|
||||
return candidate
|
||||
pytest.skip("A working bash executable is required for launcher regression coverage")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("launcher", LAUNCHERS, ids=lambda path: path.name)
|
||||
def test_online_launcher_defaults_gateway_to_one_worker(launcher: Path) -> None:
|
||||
script = _launcher_text(launcher)
|
||||
|
||||
assert 'GATEWAY_WORKERS="${GATEWAY_WORKERS:-${env_gateway_workers:-1}}"' in script
|
||||
assert '--workers "${GATEWAY_WORKERS:-1}"' in script
|
||||
assert "${GATEWAY_WORKERS:-8}" not in script
|
||||
assert "${GATEWAY_WORKERS:-2}" not in script
|
||||
|
||||
|
||||
@pytest.mark.parametrize("launcher", LAUNCHERS, ids=lambda path: path.name)
|
||||
def test_online_launcher_guards_process_local_stream_state_before_start(launcher: Path) -> None:
|
||||
script = _launcher_text(launcher)
|
||||
|
||||
guard = script.index('if [ "${GATEWAY_WORKERS}" -gt 1 ]')
|
||||
container_start = script.index("docker image inspect")
|
||||
assert guard < container_start
|
||||
assert "DEERFLOW_ALLOW_UNSAFE_MULTIWORKER" in script[guard:container_start]
|
||||
assert "MemoryStreamBridge/RunManager" in script[guard:container_start]
|
||||
assert "Location/join reconnects" in script[guard:container_start]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("launcher", LAUNCHERS, ids=lambda path: path.name)
|
||||
def test_online_launcher_forwards_validated_worker_count_to_container(launcher: Path) -> None:
|
||||
script = _launcher_text(launcher)
|
||||
|
||||
assert '-e GATEWAY_WORKERS="${GATEWAY_WORKERS}"' in script
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_launcher", LAUNCHERS, ids=lambda path: path.name)
|
||||
def test_online_launcher_rejects_env_file_multiworker_before_docker(
|
||||
tmp_path: Path,
|
||||
source_launcher: Path,
|
||||
) -> None:
|
||||
"""An old .env with eight workers must fail before touching Docker."""
|
||||
|
||||
launcher = tmp_path / "start_backend-online.sh"
|
||||
shutil.copyfile(source_launcher, launcher)
|
||||
(tmp_path / ".env").write_text(
|
||||
"AUTH_JWT_SECRET=test\n"
|
||||
"SCHEDULED_TASKS_ENABLED=false\n"
|
||||
"ELEMENT_ENABLED=false\n"
|
||||
"GATEWAY_WORKERS=8\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for relative_path in (
|
||||
"config.yaml",
|
||||
"extensions_config.json",
|
||||
"skill_evolution.json",
|
||||
"backend/pyproject.toml",
|
||||
"backend/app/gateway/app.py",
|
||||
"backend/packages/harness/deerflow/runtime/scheduler/element.py",
|
||||
):
|
||||
path = tmp_path / relative_path
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("# test fixture\n", encoding="utf-8")
|
||||
for relative_dir in ("skills", "wheelhouse", "backend/knowledge/knowledge_base"):
|
||||
(tmp_path / relative_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.pop("GATEWAY_WORKERS", None)
|
||||
env.pop("DEERFLOW_ALLOW_UNSAFE_MULTIWORKER", None)
|
||||
result = subprocess.run(
|
||||
[_working_bash(), str(launcher)],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "GATEWAY_WORKERS=8 is unsafe" in result.stderr
|
||||
assert "command not found: docker" not in result.stderr
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
"""Diagnostics must not change provider outcomes or cause accidental retries."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
from langchain_core.outputs import ChatGenerationChunk
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models import patched_openai
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broken_diagnostic_logs(monkeypatch):
|
||||
def fail(*args, **kwargs):
|
||||
raise OSError("synthetic log destination failure")
|
||||
|
||||
for level in ("info", "warning", "debug"):
|
||||
monkeypatch.setattr(patched_openai.logger, level, fail)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("outcome", ["success", "provider_error", "cancelled"])
|
||||
def test_log_failure_preserves_stream_outcome(monkeypatch, broken_diagnostic_logs, asynchronous, outcome):
|
||||
failure = asyncio.CancelledError() if outcome == "cancelled" else TimeoutError("synthetic provider error")
|
||||
chunk = ChatGenerationChunk(message=AIMessageChunk(content="answer"))
|
||||
|
||||
def parent_stream(self, *args, **kwargs):
|
||||
yield chunk
|
||||
if outcome != "success":
|
||||
raise failure
|
||||
|
||||
async def parent_astream(self, *args, **kwargs):
|
||||
yield chunk
|
||||
if outcome != "success":
|
||||
raise failure
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_stream", parent_stream)
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", parent_astream)
|
||||
model = patched_openai.PatchedChatOpenAI(model="test-model", api_key="synthetic-test-key")
|
||||
|
||||
async def collect_async():
|
||||
return [item async for item in model._astream([])]
|
||||
|
||||
def collect():
|
||||
return asyncio.run(collect_async()) if asynchronous else list(model._stream([]))
|
||||
|
||||
if outcome == "success":
|
||||
assert collect() == [chunk]
|
||||
else:
|
||||
with pytest.raises(type(failure)) as captured:
|
||||
collect()
|
||||
assert captured.value is failure
|
||||
assert patched_openai._ACTIVE_TOOL_STREAM.get() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_log_failure_preserves_cleanup_error(monkeypatch, broken_diagnostic_logs, asynchronous):
|
||||
failure = ValueError("synthetic cleanup error")
|
||||
|
||||
class ClosingIterator:
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
raise StopIteration
|
||||
|
||||
def close(self):
|
||||
raise failure
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
raise StopAsyncIteration
|
||||
|
||||
async def aclose(self):
|
||||
raise failure
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_stream", lambda *args, **kwargs: ClosingIterator())
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", lambda *args, **kwargs: ClosingIterator())
|
||||
model = patched_openai.PatchedChatOpenAI(model="test-model", api_key="synthetic-test-key")
|
||||
|
||||
async def collect_async():
|
||||
return [item async for item in model._astream([])]
|
||||
|
||||
with pytest.raises(ValueError) as captured:
|
||||
if asynchronous:
|
||||
asyncio.run(collect_async())
|
||||
else:
|
||||
list(model._stream([]))
|
||||
assert captured.value is failure
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""Safe provider error strings must retain in-process retry classification."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware
|
||||
from deerflow.agents.middlewares.llm_error_handling_middleware import LLM_ERROR_MARKER, LLMErrorHandlingMiddleware
|
||||
from deerflow.models.context_recovery import is_context_length_error
|
||||
from deerflow.models.patched_openai import (
|
||||
ArtifactRendererTransientProtocolError,
|
||||
OpenAICompatibleUpstreamError,
|
||||
PatchedChatOpenAI,
|
||||
)
|
||||
|
||||
|
||||
def middleware():
|
||||
return LLMErrorHandlingMiddleware(app_config=SimpleNamespace(circuit_breaker=SimpleNamespace(failure_threshold=3, recovery_timeout_sec=30)))
|
||||
|
||||
|
||||
def test_artifact_renderer_precommit_truncation_is_retriable() -> None:
|
||||
exc = ArtifactRendererTransientProtocolError(
|
||||
"Artifact renderer stream ended without a terminal frame"
|
||||
)
|
||||
assert middleware()._classify_error(exc) == (True, "upstream")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", ["invalid_api_key", "permission_denied", "unauthorized"])
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_code_only_auth_failure_is_not_retried(code, asynchronous, caplog):
|
||||
errors = middleware()
|
||||
exc = OpenAICompatibleUpstreamError(code=code, message="Request failed")
|
||||
calls = 0
|
||||
|
||||
def handler(request):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise exc
|
||||
|
||||
async def async_handler(request):
|
||||
return handler(request)
|
||||
|
||||
result = asyncio.run(errors.awrap_model_call(None, async_handler)) if asynchronous else errors.wrap_model_call(None, handler)
|
||||
assert calls == 1
|
||||
assert result.additional_kwargs[LLM_ERROR_MARKER] == "auth"
|
||||
assert errors._circuit_failure_count == 0
|
||||
assert "Request failed" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("message", ["maximum context length exceeded", "input is too long", "上下文长度超过限制"])
|
||||
def test_numeric_vendor_context_error_remains_detectable_without_exposing_detail(message):
|
||||
secret = "PRIVATE_PROVIDER_RESPONSE"
|
||||
exc = OpenAICompatibleUpstreamError(code=41301, message=f"{message}: {secret}")
|
||||
assert is_context_length_error(exc)
|
||||
assert middleware()._classify_error(exc) == (False, "context_length")
|
||||
assert secret not in str(exc)
|
||||
assert message not in str(exc)
|
||||
assert is_context_length_error(ValueError(message)) # Standard errors remain supported.
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_http_200_numeric_context_error_recovers_through_real_model_adapter(asynchronous, streaming, caplog):
|
||||
payloads = []
|
||||
secret = "PRIVATE_PROVIDER_RESPONSE"
|
||||
|
||||
def transport(request):
|
||||
payload = json.loads(request.content)
|
||||
payloads.append(payload)
|
||||
if len(payloads) == 1:
|
||||
body = {"choices": None, "base_resp": {"status_code": 41301, "status_msg": f"maximum context length exceeded: {secret}"}}
|
||||
if payload.get("stream"):
|
||||
return httpx.Response(200, content=("data: " + json.dumps(body) + "\n\n").encode(), headers={"content-type": "text/event-stream"})
|
||||
return httpx.Response(200, json=body)
|
||||
choice = {"index": 0, "finish_reason": "stop"}
|
||||
body = {"id": "test-recovery", "created": 0, "model": "test-model", "choices": [choice]}
|
||||
if payload.get("stream"):
|
||||
body["object"] = "chat.completion.chunk"
|
||||
choice["delta"] = {"role": "assistant", "content": "recovered"}
|
||||
return httpx.Response(200, content=("data: " + json.dumps(body) + "\n\ndata: [DONE]\n\n").encode(), headers={"content-type": "text/event-stream"})
|
||||
body["object"] = "chat.completion"
|
||||
choice["message"] = {"role": "assistant", "content": "recovered"}
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
model = PatchedChatOpenAI(
|
||||
model="test-model",
|
||||
api_key="synthetic-test-key",
|
||||
base_url="https://local.invalid/v1",
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(transport)),
|
||||
http_async_client=httpx.AsyncClient(transport=httpx.MockTransport(transport)),
|
||||
)
|
||||
errors = middleware()
|
||||
agent = create_agent(model=model, middleware=[errors, ContextWindowGuardMiddleware()])
|
||||
inputs = {"messages": [HumanMessage(content="Summarize: " + "material " * 10000)]}
|
||||
|
||||
async def collect():
|
||||
return [event async for event in agent.astream(inputs, stream_mode="messages")]
|
||||
|
||||
if streaming:
|
||||
result = asyncio.run(collect()) if asynchronous else list(agent.stream(inputs, stream_mode="messages"))
|
||||
assert "recovered" in str(result)
|
||||
else:
|
||||
result = asyncio.run(agent.ainvoke(inputs)) if asynchronous else agent.invoke(inputs)
|
||||
assert result["messages"][-1].content == "recovered"
|
||||
assert len(payloads) == 2
|
||||
assert len(json.dumps(payloads[1])) < len(json.dumps(payloads[0]))
|
||||
assert errors._circuit_failure_count == 0
|
||||
assert secret not in caplog.text
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
|
||||
from deerflow.models.patched_openai import (
|
||||
_append_reasoning_content,
|
||||
_extract_reasoning_content,
|
||||
_restore_reasoning_content,
|
||||
)
|
||||
|
||||
|
||||
def test_extracts_legacy_reasoning_content():
|
||||
assert _extract_reasoning_content({"reasoning_content": "legacy"}) == "legacy"
|
||||
|
||||
|
||||
def test_extracts_vllm_reasoning():
|
||||
assert _extract_reasoning_content({"reasoning": "new spelling"}) == "new spelling"
|
||||
|
||||
|
||||
def test_normalizes_vllm_reasoning_for_existing_clients():
|
||||
message = AIMessageChunk(content="")
|
||||
|
||||
updated = _append_reasoning_content(message, "thinking", reasoning="thinking")
|
||||
|
||||
assert updated.additional_kwargs["reasoning_content"] == "thinking"
|
||||
assert updated.additional_kwargs["reasoning"] == "thinking"
|
||||
|
||||
|
||||
def test_restores_original_vllm_reasoning_spelling_for_follow_up_turn():
|
||||
original = AIMessage(
|
||||
content="answer",
|
||||
additional_kwargs={
|
||||
"reasoning": "thinking",
|
||||
"reasoning_content": "thinking",
|
||||
},
|
||||
)
|
||||
payload = {"role": "assistant", "content": "answer"}
|
||||
|
||||
_restore_reasoning_content(payload, original)
|
||||
|
||||
assert payload["reasoning"] == "thinking"
|
||||
assert "reasoning_content" not in payload
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
"""Replay provider SSE locally; reasoning must arrive before the answer exists."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from deerflow.models.patched_openai import PatchedChatOpenAI
|
||||
from deerflow.runtime.serialization import serialize
|
||||
|
||||
|
||||
def _event(choice: dict, *, root_fields: dict | None = None) -> bytes:
|
||||
payload = {
|
||||
"id": "chatcmpl-local-replay",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "local-replay",
|
||||
"choices": [{"index": 0, "finish_reason": None, **choice}],
|
||||
**(root_fields or {}),
|
||||
}
|
||||
return f"data: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
class _GatedProviderStream(httpx.AsyncByteStream):
|
||||
def __init__(self, first_choice: dict, release_answer: asyncio.Event, *, root_fields: dict | None = None) -> None:
|
||||
self.first_choice = first_choice
|
||||
self.release_answer = release_answer
|
||||
self.closed = False
|
||||
self.root_fields = root_fields
|
||||
|
||||
async def __aiter__(self):
|
||||
yield _event(self.first_choice, root_fields=self.root_fields)
|
||||
# There is intentionally no answer until the test receives reasoning.
|
||||
await self.release_answer.wait()
|
||||
yield _event({"delta": {"content": "local answer"}})
|
||||
yield _event({"delta": {}, "finish_reason": "stop"})
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("field", ["reasoning_content", "reasoning"])
|
||||
@pytest.mark.parametrize("container", ["delta", "message", "message_with_empty_delta"])
|
||||
async def test_sdk_yields_reasoning_before_provider_answer(field, container):
|
||||
release_answer = asyncio.Event()
|
||||
first_choice = {
|
||||
"message" if container == "message_with_empty_delta" else container:
|
||||
{"role": "assistant", "content": None, field: "local reasoning"},
|
||||
}
|
||||
if container == "message_with_empty_delta":
|
||||
first_choice["delta"] = {}
|
||||
provider = _GatedProviderStream(
|
||||
first_choice,
|
||||
release_answer,
|
||||
)
|
||||
|
||||
async def handler(request):
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=provider)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
model = PatchedChatOpenAI(
|
||||
model="local-replay", api_key="local-unused", base_url="https://local.invalid/v1",
|
||||
http_async_client=client, max_retries=0,
|
||||
)
|
||||
stream = model.astream("local input")
|
||||
try:
|
||||
first = await asyncio.wait_for(anext(stream), timeout=2)
|
||||
assert not release_answer.is_set()
|
||||
assert first.additional_kwargs["reasoning_content"] == "local reasoning"
|
||||
serialized = serialize((first, {"langgraph_node": "model"}), mode="messages")
|
||||
assert serialized[0]["additional_kwargs"]["reasoning_content"] == "local reasoning"
|
||||
release_answer.set()
|
||||
rest = [chunk async for chunk in stream]
|
||||
assert "".join(chunk.content for chunk in rest) == "local answer"
|
||||
finally:
|
||||
release_answer.set()
|
||||
await stream.aclose()
|
||||
assert provider.closed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_empty_error_object_does_not_abort_first_reasoning():
|
||||
release_answer = asyncio.Event()
|
||||
provider = _GatedProviderStream(
|
||||
{"delta": {"role": "assistant", "reasoning_content": "local reasoning"}},
|
||||
release_answer,
|
||||
root_fields={"error": {}},
|
||||
)
|
||||
|
||||
async def handler(request):
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=provider)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
model = PatchedChatOpenAI(
|
||||
model="local-replay", api_key="local-unused", base_url="https://local.invalid/v1",
|
||||
http_async_client=client, max_retries=0,
|
||||
)
|
||||
stream = model.astream("local input")
|
||||
try:
|
||||
first = await asyncio.wait_for(anext(stream), timeout=2)
|
||||
assert first.additional_kwargs["reasoning_content"] == "local reasoning"
|
||||
finally:
|
||||
release_answer.set()
|
||||
await stream.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_model_delivers_twenty_reasoning_streams_before_answers():
|
||||
release_answer = asyncio.Event()
|
||||
providers: list[_GatedProviderStream] = []
|
||||
|
||||
async def handler(request):
|
||||
index = len(providers)
|
||||
provider = _GatedProviderStream(
|
||||
{"delta": {"role": "assistant", "reasoning_content": f"local reasoning {index}"}},
|
||||
release_answer,
|
||||
)
|
||||
providers.append(provider)
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=provider)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
model = PatchedChatOpenAI(
|
||||
model="local-replay", api_key="local-unused", base_url="https://local.invalid/v1",
|
||||
http_async_client=client, max_retries=0,
|
||||
)
|
||||
streams = [model.astream(f"local input {index}") for index in range(20)]
|
||||
try:
|
||||
first = await asyncio.wait_for(asyncio.gather(*(anext(stream) for stream in streams)), timeout=5)
|
||||
assert len({chunk.additional_kwargs["reasoning_content"] for chunk in first}) == 20
|
||||
assert not release_answer.is_set()
|
||||
release_answer.set()
|
||||
|
||||
async def consume(stream):
|
||||
return [chunk async for chunk in stream]
|
||||
|
||||
completed = await asyncio.wait_for(asyncio.gather(*(consume(stream) for stream in streams)), timeout=5)
|
||||
assert all("".join(chunk.content for chunk in chunks) == "local answer" for chunks in completed)
|
||||
finally:
|
||||
release_answer.set()
|
||||
await asyncio.gather(*(stream.aclose() for stream in streams))
|
||||
assert len(providers) == 20
|
||||
assert all(provider.closed for provider in providers)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("flag,expected", [(True, "true"), (False, "false"), ("PRIVATE_FLAG", "other")])
|
||||
async def test_reasoning_summary_has_counts_but_no_request_or_response_content(flag, expected, caplog):
|
||||
release_answer = asyncio.Event()
|
||||
release_answer.set()
|
||||
provider = _GatedProviderStream(
|
||||
{"delta": {}, "message": {"reasoning": "PRIVATE_REASONING"}}, release_answer,
|
||||
)
|
||||
|
||||
async def handler(request):
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=provider)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
model = PatchedChatOpenAI(
|
||||
model="local-replay", api_key="PRIVATE_API_KEY", base_url="https://local.invalid/v1",
|
||||
http_async_client=client, max_retries=0, extra_body={"enable_thinking": flag},
|
||||
)
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
chunks = [chunk async for chunk in model.astream("PRIVATE_PROMPT")]
|
||||
assert chunks[0].additional_kwargs["reasoning_content"] == "PRIVATE_REASONING"
|
||||
summaries = [record.getMessage() for record in caplog.records if "OPENAI_TOOL_STREAM_SUMMARY" in record.getMessage()]
|
||||
assert len(summaries) == 1
|
||||
assert "incoming_reasoning_chunks=1" in summaries[0]
|
||||
assert "emitted_reasoning_chunks=1" in summaries[0]
|
||||
assert "first_reasoning_ms=-1" not in summaries[0]
|
||||
assert f"request_enable_thinking={expected}" in summaries[0]
|
||||
assert "PRIVATE_" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("argument_mode", ["delta", "cumulative"])
|
||||
@pytest.mark.parametrize("repeat_tool_name", [False, True])
|
||||
@pytest.mark.parametrize("empty_header", [False, True])
|
||||
async def test_sdk_yields_partial_write_file_arguments_before_json_finishes(argument_mode, repeat_tool_name, empty_header):
|
||||
release_arguments = asyncio.Event()
|
||||
expected_arguments = {"path": "/mnt/user-data/outputs/replay.md", "content": "# Heading\n\nMarkdown details\n"}
|
||||
full_arguments = json.dumps(expected_arguments, separators=(",", ":"))
|
||||
split = full_arguments.index("Markdown")
|
||||
partial_arguments = full_arguments[:split]
|
||||
|
||||
class ToolStream(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
if empty_header:
|
||||
yield _event({"delta": {"role": "assistant", "tool_calls": [{
|
||||
"index": 0, "id": "call_write", "type": "function",
|
||||
"function": {"name": "write_file", "arguments": ""},
|
||||
}]}})
|
||||
yield _event({"delta": {"role": "assistant", "tool_calls": [{
|
||||
"index": 0, "id": "call_write", "type": "function",
|
||||
"function": {"name": "write_file", "arguments": partial_arguments},
|
||||
}]}})
|
||||
await release_arguments.wait()
|
||||
continuation = {
|
||||
"index": 0, "function": {"arguments": full_arguments if argument_mode == "cumulative" else full_arguments[split:]},
|
||||
}
|
||||
if repeat_tool_name:
|
||||
continuation["id"] = "call_write"
|
||||
continuation["function"]["name"] = "write_file"
|
||||
yield _event({"delta": {"tool_calls": [continuation]}})
|
||||
yield _event({"delta": {}, "finish_reason": "tool_calls"})
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
async def handler(request):
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=ToolStream())
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
model = PatchedChatOpenAI(
|
||||
model="local-replay", api_key="local-unused", base_url="https://local.invalid/v1",
|
||||
http_async_client=client, max_retries=0,
|
||||
)
|
||||
stream = model.bind_tools([{
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "parameters": {"type": "object", "properties": {}}},
|
||||
}]).astream("local input")
|
||||
try:
|
||||
header = await asyncio.wait_for(anext(stream), timeout=2) if empty_header else None
|
||||
first = await asyncio.wait_for(anext(stream), timeout=2)
|
||||
assert not release_arguments.is_set()
|
||||
assert first.tool_call_chunks[0]["args"] == partial_arguments
|
||||
# An unfinished JSON string is visible to the live-preview client.
|
||||
preview = header + first if header is not None else first
|
||||
assert preview.tool_calls[0]["name"] == "write_file"
|
||||
assert preview.tool_calls[0]["args"]["content"] == "# Heading\n\n"
|
||||
serialized = serialize((first, {"langgraph_node": "model"}), mode="messages")
|
||||
assert serialized[0]["tool_call_chunks"][0]["args"] == partial_arguments
|
||||
release_arguments.set()
|
||||
combined = preview
|
||||
async for chunk in stream:
|
||||
combined += chunk
|
||||
assert combined.tool_calls == [{"name": "write_file", "args": expected_arguments, "id": "call_write", "type": "tool_call"}]
|
||||
finally:
|
||||
release_arguments.set()
|
||||
await stream.aclose()
|
||||
+772
@@ -0,0 +1,772 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from deerflow.models.patched_openai import (
|
||||
PatchedChatOpenAI,
|
||||
_restore_tool_call_signatures,
|
||||
_safe_completion_id,
|
||||
rewrite_cumulative_tool_call_arguments,
|
||||
)
|
||||
|
||||
|
||||
def _chunk(arguments: str, *, index: int = 0) -> dict:
|
||||
return {
|
||||
"id": "chatcmpl-1",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": index,
|
||||
"id": "call_1",
|
||||
"function": {
|
||||
"name": "write_file",
|
||||
"arguments": arguments,
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw_id", "expected"),
|
||||
[
|
||||
("chatcmpl-safe_1.2", "chatcmpl-safe_1.2"),
|
||||
("https://model.example/SECRET_BODY", "none"),
|
||||
("chatcmpl-safe\nSECRET_BODY", "none"),
|
||||
("SECRET_BODY\n", "none"),
|
||||
("x" * 81, "none"),
|
||||
(123, "none"),
|
||||
],
|
||||
)
|
||||
def test_provider_completion_id_is_strictly_sanitized(raw_id, expected):
|
||||
assert _safe_completion_id(raw_id) == expected
|
||||
|
||||
|
||||
def test_rewrites_growing_argument_snapshots_into_deltas():
|
||||
snapshots: dict[int, str] = {}
|
||||
first = _chunk('{"path": "/mnt/user-data/outputs/page.html"')
|
||||
rewrite_cumulative_tool_call_arguments(first, snapshots)
|
||||
assert first["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == ('{"path": "/mnt/user-data/outputs/page.html"')
|
||||
|
||||
second = _chunk('{"path": "/mnt/user-data/outputs/page.html", "content": "<!doctype html>')
|
||||
rewrite_cumulative_tool_call_arguments(second, snapshots)
|
||||
assert second["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == (', "content": "<!doctype html>')
|
||||
|
||||
|
||||
def test_leaves_true_openai_argument_deltas_unchanged():
|
||||
snapshots: dict[int, str] = {}
|
||||
first = _chunk('{"path":')
|
||||
rewrite_cumulative_tool_call_arguments(first, snapshots)
|
||||
second = _chunk(' "/mnt/user-data/outputs/page.html", "content": "<p>hi"')
|
||||
rewrite_cumulative_tool_call_arguments(second, snapshots)
|
||||
|
||||
assert second["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == (' "/mnt/user-data/outputs/page.html", "content": "<p>hi"')
|
||||
assert snapshots[0].startswith('{"path": "/mnt/user-data/outputs/page.html"')
|
||||
|
||||
|
||||
def test_duplicate_argument_snapshots_emit_empty_delta():
|
||||
snapshots: dict[int, str] = {}
|
||||
payload = '{"path": "/mnt/user-data/outputs/page.html", "content": "<html></html>"}'
|
||||
first = _chunk(payload)
|
||||
rewrite_cumulative_tool_call_arguments(first, snapshots)
|
||||
second = _chunk(payload)
|
||||
rewrite_cumulative_tool_call_arguments(second, snapshots)
|
||||
|
||||
assert second["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pieces",
|
||||
[
|
||||
['{"outer":', "{", '"x":1}}'],
|
||||
['{"content":"<style>', "{", 'color:red}</style>"}'],
|
||||
],
|
||||
)
|
||||
def test_standard_delta_brace_fragments_round_trip(pieces):
|
||||
snapshots: dict[int, str] = {}
|
||||
argument_modes: dict[int, str] = {}
|
||||
emitted: list[str] = []
|
||||
|
||||
for piece in pieces:
|
||||
chunk = _chunk(piece)
|
||||
rewrite_cumulative_tool_call_arguments(
|
||||
chunk,
|
||||
snapshots,
|
||||
argument_modes=argument_modes,
|
||||
)
|
||||
emitted.append(chunk["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"])
|
||||
|
||||
assert "".join(emitted) == "".join(pieces)
|
||||
assert argument_modes == {0: "delta"}
|
||||
|
||||
|
||||
def test_new_call_id_reusing_index_resets_snapshot():
|
||||
snapshots: dict[int, str] = {}
|
||||
call_ids: dict[int, str] = {}
|
||||
first = _chunk('{"path":"one"}')
|
||||
rewrite_cumulative_tool_call_arguments(first, snapshots, call_ids=call_ids)
|
||||
|
||||
second = _chunk('{"path":"two"}')
|
||||
second["choices"][0]["delta"]["tool_calls"][0]["id"] = "call_2"
|
||||
rewrite_cumulative_tool_call_arguments(second, snapshots, call_ids=call_ids)
|
||||
|
||||
assert second["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == '{"path":"two"}'
|
||||
assert snapshots == {0: '{"path":"two"}'}
|
||||
|
||||
|
||||
def test_repeated_tool_names_are_isolated_by_index_and_reset_by_call_id():
|
||||
snapshots: dict[int, str] = {}
|
||||
call_ids: dict[int, str] = {}
|
||||
names: dict[int, str] = {}
|
||||
|
||||
def convert(index, call_id, name, arguments):
|
||||
chunk = _chunk(arguments, index=index)
|
||||
call = chunk["choices"][0]["delta"]["tool_calls"][0]
|
||||
call["id"] = call_id
|
||||
call["function"]["name"] = name
|
||||
rewrite_cumulative_tool_call_arguments(chunk, snapshots, call_ids=call_ids, tool_name_snapshots=names)
|
||||
return call["function"]["name"]
|
||||
|
||||
assert convert(0, "write-1", "write_file", '{"path":') == "write_file"
|
||||
assert convert(1, "read-1", "read_file", '{"path":') == "read_file"
|
||||
assert convert(0, "write-1", "write_file", '"one"}') is None
|
||||
assert convert(1, "read-1", "read_file", '"two"}') is None
|
||||
assert names == {0: "write_file", 1: "read_file"}
|
||||
assert convert(0, "write-2", "write_file", '{"path":"new"}') == "write_file"
|
||||
assert names == {0: "write_file", 1: "read_file"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parts,expected", [(["write_", "file"], "write_file"), (["foo", "foo"], "foofoo")])
|
||||
def test_legitimate_fragmented_function_name_is_not_deduplicated(parts, expected):
|
||||
snapshots: dict[int, str] = {}
|
||||
names: dict[int, str] = {}
|
||||
emitted = []
|
||||
for name, arguments in zip(parts, ["", '{"path":"one"}']):
|
||||
chunk = _chunk(arguments)
|
||||
function = chunk["choices"][0]["delta"]["tool_calls"][0]["function"]
|
||||
function["name"] = name
|
||||
rewrite_cumulative_tool_call_arguments(chunk, snapshots, tool_name_snapshots=names)
|
||||
emitted.append(function["name"])
|
||||
assert "".join(emitted) == expected
|
||||
assert names == {0: expected}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw_index", [None, "not-a-number"])
|
||||
def test_missing_or_invalid_tool_index_is_normalized_in_place(raw_index):
|
||||
chunk = _chunk('{"path":"/mnt/user-data/outputs/test.md"}')
|
||||
tool_call = chunk["choices"][0]["delta"]["tool_calls"][0]
|
||||
if raw_index is None:
|
||||
tool_call.pop("index")
|
||||
else:
|
||||
tool_call["index"] = raw_index
|
||||
|
||||
rewrite_cumulative_tool_call_arguments(chunk, {})
|
||||
|
||||
assert tool_call["index"] == 0
|
||||
|
||||
|
||||
def test_multiple_missing_indexes_use_stable_positions():
|
||||
chunk = _chunk('{"path":"one"}')
|
||||
first_call = chunk["choices"][0]["delta"]["tool_calls"][0]
|
||||
first_call.pop("index")
|
||||
second_call = {
|
||||
"id": "call_2",
|
||||
"function": {"name": "write_file", "arguments": '{"path":"two"}'},
|
||||
}
|
||||
chunk["choices"][0]["delta"]["tool_calls"].append(second_call)
|
||||
|
||||
rewrite_cumulative_tool_call_arguments(chunk, {})
|
||||
|
||||
assert first_call["index"] == 0
|
||||
assert second_call["index"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw_index", [None, "not-a-number"])
|
||||
def test_staggered_invalid_indexes_allocate_stable_slots_by_id(raw_index):
|
||||
snapshots: dict[int, str] = {}
|
||||
call_ids: dict[int, str] = {}
|
||||
argument_modes: dict[int, str] = {}
|
||||
|
||||
first = _chunk('{"first":')
|
||||
first_call = first["choices"][0]["delta"]["tool_calls"][0]
|
||||
if raw_index is None:
|
||||
first_call.pop("index")
|
||||
else:
|
||||
first_call["index"] = raw_index
|
||||
rewrite_cumulative_tool_call_arguments(
|
||||
first,
|
||||
snapshots,
|
||||
call_ids=call_ids,
|
||||
argument_modes=argument_modes,
|
||||
)
|
||||
|
||||
second = _chunk('{"second":2}')
|
||||
second_call = second["choices"][0]["delta"]["tool_calls"][0]
|
||||
if raw_index is None:
|
||||
second_call.pop("index")
|
||||
else:
|
||||
second_call["index"] = raw_index
|
||||
second_call["id"] = "call_2"
|
||||
rewrite_cumulative_tool_call_arguments(
|
||||
second,
|
||||
snapshots,
|
||||
call_ids=call_ids,
|
||||
argument_modes=argument_modes,
|
||||
)
|
||||
|
||||
first_continuation = _chunk('{"first":1}')
|
||||
continued_call = first_continuation["choices"][0]["delta"]["tool_calls"][0]
|
||||
if raw_index is None:
|
||||
continued_call.pop("index")
|
||||
else:
|
||||
continued_call["index"] = raw_index
|
||||
rewrite_cumulative_tool_call_arguments(
|
||||
first_continuation,
|
||||
snapshots,
|
||||
call_ids=call_ids,
|
||||
argument_modes=argument_modes,
|
||||
)
|
||||
|
||||
assert first_call["index"] == 0
|
||||
assert second_call["index"] == 1
|
||||
assert continued_call["index"] == 0
|
||||
assert continued_call["function"]["arguments"] == "1}"
|
||||
assert call_ids == {0: "call_1", 1: "call_2"}
|
||||
|
||||
|
||||
def test_full_message_and_structured_arguments_are_normalized():
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
chunk = {
|
||||
"id": "chatcmpl-message-shape",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "write_file",
|
||||
"arguments": {"path": "/mnt/user-data/outputs/test.md", "content": "safe"},
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
result = model._convert_chunk_to_generation_chunk(chunk, AIMessageChunk, None)
|
||||
|
||||
assert result is not None
|
||||
assert result.message.tool_call_chunks[0]["args"] == ('{"path":"/mnt/user-data/outputs/test.md","content":"safe"}')
|
||||
|
||||
|
||||
def test_singleton_tool_call_object_is_normalized_to_list():
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
chunk = _chunk('{"path":"/mnt/user-data/outputs/test.md"}')
|
||||
delta = chunk["choices"][0]["delta"]
|
||||
delta["tool_calls"] = delta["tool_calls"][0]
|
||||
|
||||
result = model._convert_chunk_to_generation_chunk(chunk, AIMessageChunk, None)
|
||||
|
||||
assert result is not None
|
||||
assert result.message.tool_call_chunks[0]["name"] == "write_file"
|
||||
assert result.message.tool_call_chunks[0]["index"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["message", "root"])
|
||||
def test_empty_delta_uses_compatible_tool_call_container(source):
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
tool_calls = _chunk("{}")["choices"][0]["delta"]["tool_calls"]
|
||||
choice = {"index": 0, "delta": {}}
|
||||
if source == "message":
|
||||
choice["message"] = {"role": "assistant", "tool_calls": tool_calls}
|
||||
else:
|
||||
choice["tool_calls"] = tool_calls
|
||||
|
||||
result = model._convert_chunk_to_generation_chunk(
|
||||
{"id": "chatcmpl-empty-delta", "choices": [choice]},
|
||||
AIMessageChunk,
|
||||
None,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.message.tool_call_chunks[0]["name"] == "write_file"
|
||||
|
||||
|
||||
def test_standard_delta_tool_calls_are_not_overwritten_by_root_fallback():
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
choice = _chunk("{}")["choices"][0]
|
||||
choice["tool_calls"] = [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "wrong-call",
|
||||
"function": {"name": "wrong_tool", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
|
||||
result = model._convert_chunk_to_generation_chunk(
|
||||
{"id": "chatcmpl-standard-wins", "choices": [choice]},
|
||||
AIMessageChunk,
|
||||
None,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.message.tool_call_chunks[0]["name"] == "write_file"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("signature_fields", "expected_path", "expected_signature"),
|
||||
[
|
||||
({"thought_signature": "sig-snake"}, ("thought_signature",), "sig-snake"),
|
||||
({"thoughtSignature": "sig-camel"}, ("thoughtSignature",), "sig-camel"),
|
||||
(
|
||||
{"extra_content": {"google": {"thought_signature": "sig-google"}}},
|
||||
("extra_content", "google", "thought_signature"),
|
||||
"sig-google",
|
||||
),
|
||||
(
|
||||
{"extra_content": {"google": {"thoughtSignature": "sig-google-camel"}}},
|
||||
("extra_content", "google", "thoughtSignature"),
|
||||
"sig-google-camel",
|
||||
),
|
||||
(
|
||||
{"extraContent": {"google": {"thought_signature": "sig-extra-camel"}}},
|
||||
("extraContent", "google", "thought_signature"),
|
||||
"sig-extra-camel",
|
||||
),
|
||||
(
|
||||
{"extraContent": {"google": {"thoughtSignature": "sig-all-camel"}}},
|
||||
("extraContent", "google", "thoughtSignature"),
|
||||
"sig-all-camel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_non_stream_thought_signature_survives_next_request(
|
||||
signature_fields,
|
||||
expected_path,
|
||||
expected_signature,
|
||||
):
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
raw_tool_call = {
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "arguments": "{}"},
|
||||
**signature_fields,
|
||||
}
|
||||
response = {
|
||||
"id": "chatcmpl-signature",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": None, "tool_calls": [raw_tool_call]},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
message = model._create_chat_result(response).generations[0].message
|
||||
payload_tool_call = model._get_request_payload([message])["messages"][0]["tool_calls"][0]
|
||||
|
||||
value = payload_tool_call
|
||||
for key in expected_path:
|
||||
value = value[key]
|
||||
assert value == expected_signature
|
||||
|
||||
|
||||
def test_stream_signature_association_prefers_tool_call_id():
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
chunk = _chunk("{}")
|
||||
tool_calls = chunk["choices"][0]["delta"]["tool_calls"]
|
||||
tool_calls[0]["thought_signature"] = "sig-one"
|
||||
tool_calls.append(
|
||||
{
|
||||
"index": 1,
|
||||
"id": "call_2",
|
||||
"function": {"name": "other_tool", "arguments": "{}"},
|
||||
"thoughtSignature": "sig-two",
|
||||
}
|
||||
)
|
||||
|
||||
generation = model._convert_chunk_to_generation_chunk(chunk, AIMessageChunk, None)
|
||||
assert generation is not None
|
||||
message = generation.message.model_copy(update={"tool_calls": list(reversed(generation.message.tool_calls))})
|
||||
payload_tool_calls = model._get_request_payload([message])["messages"][0]["tool_calls"]
|
||||
payload_by_id = {tool_call["id"]: tool_call for tool_call in payload_tool_calls}
|
||||
|
||||
assert payload_by_id["call_1"]["thought_signature"] == "sig-one"
|
||||
assert payload_by_id["call_2"]["thoughtSignature"] == "sig-two"
|
||||
|
||||
|
||||
def test_repeated_stream_signature_survives_chunk_aggregation_exactly():
|
||||
model = PatchedChatOpenAI(model="test-model", api_key="test-key")
|
||||
first = _chunk('{"path":')
|
||||
second = _chunk('"test.md"}')
|
||||
for chunk in (first, second):
|
||||
chunk["choices"][0]["delta"]["tool_calls"][0]["thought_signature"] = "opaque-signature"
|
||||
|
||||
first_generation = model._convert_chunk_to_generation_chunk(first, AIMessageChunk, None)
|
||||
second_generation = model._convert_chunk_to_generation_chunk(second, AIMessageChunk, None)
|
||||
assert first_generation is not None
|
||||
assert second_generation is not None
|
||||
|
||||
merged = first_generation + second_generation
|
||||
payload_tool_call = model._get_request_payload([merged.message])["messages"][0]["tool_calls"][0]
|
||||
|
||||
assert payload_tool_call["thought_signature"] == "opaque-signature"
|
||||
|
||||
|
||||
def test_signature_association_falls_back_to_index_without_cross_call_leakage():
|
||||
signed = AIMessage(
|
||||
content="",
|
||||
additional_kwargs={
|
||||
"_deerflow_tool_call_signatures": [
|
||||
{"tool_index": 0, "location": "top_snake", "signature": "sig-zero"},
|
||||
{"tool_index": 1, "location": "top_camel", "signature": "sig-one"},
|
||||
]
|
||||
},
|
||||
)
|
||||
payload = {
|
||||
"tool_calls": [
|
||||
{"type": "function", "function": {"name": "first", "arguments": "{}"}},
|
||||
{"type": "function", "function": {"name": "second", "arguments": "{}"}},
|
||||
]
|
||||
}
|
||||
_restore_tool_call_signatures(payload, signed)
|
||||
|
||||
assert payload["tool_calls"][0]["thought_signature"] == "sig-zero"
|
||||
assert payload["tool_calls"][1]["thoughtSignature"] == "sig-one"
|
||||
|
||||
unsigned_payload = {"tool_calls": [{"type": "function", "function": {"name": "next", "arguments": "{}"}}]}
|
||||
_restore_tool_call_signatures(unsigned_payload, AIMessage(content=""))
|
||||
assert "thought_signature" not in unsigned_payload["tool_calls"][0]
|
||||
|
||||
|
||||
def test_non_stream_structured_arguments_are_normalized_without_logging_body(caplog):
|
||||
secret = "NON_STREAM_SECRET_BODY"
|
||||
response_dict = {
|
||||
"id": "chatcmpl-structured",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "write_file",
|
||||
"arguments": {"path": "test.md", "content": secret},
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with warnings.catch_warnings(record=True) as captured:
|
||||
warnings.simplefilter("always")
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
result = PatchedChatOpenAI(model="test-model", api_key="test-key")._create_chat_result(response_dict)
|
||||
|
||||
assert result.generations[0].message.tool_calls[0]["args"] == {
|
||||
"path": "test.md",
|
||||
"content": secret,
|
||||
}
|
||||
assert secret not in "\n".join(str(warning.message) for warning in captured)
|
||||
assert secret not in "\n".join(record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
def test_non_stream_typed_response_dump_disables_serializer_warnings():
|
||||
secret = "NON_STREAM_SECRET_BODY"
|
||||
response_dict = {
|
||||
"id": "chatcmpl-typed",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "safe"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
class CompatibleResponse:
|
||||
def model_dump(self, *, warnings=True, **_kwargs):
|
||||
if warnings:
|
||||
import warnings as warnings_module
|
||||
|
||||
warnings_module.warn(f"Pydantic serializer warnings: {secret}", UserWarning)
|
||||
return response_dict
|
||||
|
||||
with warnings.catch_warnings(record=True) as captured:
|
||||
warnings.simplefilter("always")
|
||||
PatchedChatOpenAI(model="test-model", api_key="test-key")._create_chat_result(CompatibleResponse())
|
||||
|
||||
assert secret not in "\n".join(str(warning.message) for warning in captured)
|
||||
|
||||
|
||||
def test_real_sdk_stream_suppresses_structured_argument_serializer_warning(caplog):
|
||||
secret = "STREAM_SECRET_TOOL_BODY"
|
||||
secret_signature = "STREAM_SECRET_THOUGHT_SIGNATURE"
|
||||
events = [
|
||||
{
|
||||
"id": "chatcmpl-sdk-stream",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "write_file", "arguments": {"content": secret}},
|
||||
"thought_signature": secret_signature,
|
||||
}
|
||||
],
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-sdk-stream",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
},
|
||||
]
|
||||
body = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n"
|
||||
|
||||
def handler(_request):
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=body)
|
||||
|
||||
client = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
model = PatchedChatOpenAI(
|
||||
model="test-model",
|
||||
api_key="test-key",
|
||||
base_url="https://model.example/v1",
|
||||
http_client=client,
|
||||
)
|
||||
try:
|
||||
with warnings.catch_warnings(record=True) as captured:
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
chunks = list(model.stream("hello"))
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
assert chunks
|
||||
warning_text = "\n".join(str(warning.message) for warning in captured)
|
||||
log_text = "\n".join(record.getMessage() for record in caplog.records)
|
||||
assert "Pydantic serializer warnings" not in warning_text
|
||||
assert secret not in warning_text
|
||||
assert secret not in log_text
|
||||
assert secret_signature not in log_text
|
||||
assert "structured_args=1" in log_text
|
||||
assert "provider_completion_id=chatcmpl-sdk-stream" in log_text
|
||||
|
||||
merged = chunks[0]
|
||||
for chunk in chunks[1:]:
|
||||
merged += chunk
|
||||
payload_tool_call = model._get_request_payload([merged])["messages"][0]["tool_calls"][0]
|
||||
assert payload_tool_call["thought_signature"] == secret_signature
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_stream_logs_one_content_free_summary(monkeypatch, caplog):
|
||||
secret_path = "/mnt/user-data/outputs/DO_NOT_LOG.md"
|
||||
secret_content = "TOP_SECRET_TOOL_BODY"
|
||||
payload_1 = '{"path":"' + secret_path + '"'
|
||||
payload_2 = payload_1 + ',"content":"' + secret_content + '"}'
|
||||
raw_chunks = [
|
||||
_chunk(payload_1),
|
||||
_chunk(payload_2),
|
||||
{
|
||||
"id": "chatcmpl-1",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
},
|
||||
]
|
||||
raw_chunks[0]["choices"][0]["delta"]["tool_calls"][0]["thought_signature"] = "SECRET_SIGNATURE"
|
||||
|
||||
async def fake_parent_astream(self, *args, **kwargs):
|
||||
for raw in raw_chunks:
|
||||
converted = self._convert_chunk_to_generation_chunk(raw, AIMessageChunk, None)
|
||||
if converted is not None:
|
||||
yield converted
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", fake_parent_astream)
|
||||
model = PatchedChatOpenAI(model="safe-model", api_key="test-key")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in model._astream(
|
||||
[],
|
||||
run_manager=SimpleNamespace(run_id="run-123"),
|
||||
)
|
||||
]
|
||||
|
||||
assert chunks
|
||||
summaries = [record.getMessage() for record in caplog.records if "OPENAI_TOOL_STREAM_SUMMARY" in record.getMessage()]
|
||||
assert len(summaries) == 1
|
||||
summary = summaries[0]
|
||||
assert "model=safe-model" in summary
|
||||
assert "run_id=run-123" in summary
|
||||
assert "tool_arg_chunks=2" in summary
|
||||
assert "initial=1" in summary
|
||||
assert "cumulative=1" in summary
|
||||
assert "valid_final_json=1" in summary
|
||||
assert "invalid_final_json=0" in summary
|
||||
assert "finish_reason=tool_calls" in summary
|
||||
assert secret_path not in summary
|
||||
assert secret_content not in summary
|
||||
assert "SECRET_SIGNATURE" not in summary
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_truncated_final_tool_json_is_a_content_free_warning(monkeypatch, caplog):
|
||||
async def fake_parent_astream(self, *args, **kwargs):
|
||||
for raw in (
|
||||
_chunk('{"path":'),
|
||||
{
|
||||
"id": "chatcmpl-1",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}],
|
||||
},
|
||||
):
|
||||
converted = self._convert_chunk_to_generation_chunk(raw, AIMessageChunk, None)
|
||||
if converted is not None:
|
||||
yield converted
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", fake_parent_astream)
|
||||
model = PatchedChatOpenAI(model="safe-model", api_key="test-key")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
_ = [chunk async for chunk in model._astream([])]
|
||||
|
||||
summaries = [record for record in caplog.records if "OPENAI_TOOL_STREAM_SUMMARY" in record.getMessage()]
|
||||
assert len(summaries) == 1
|
||||
assert summaries[0].levelno == logging.WARNING
|
||||
assert "valid_final_json=0" in summaries[0].getMessage()
|
||||
assert "invalid_final_json=1" in summaries[0].getMessage()
|
||||
assert '{"path":' not in summaries[0].getMessage()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_stream_error_logs_type_without_exception_message(monkeypatch, caplog):
|
||||
async def failing_parent_astream(self, *args, **kwargs):
|
||||
if False:
|
||||
yield None
|
||||
raise TimeoutError("TOP_SECRET_EXCEPTION_DETAIL")
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", failing_parent_astream)
|
||||
model = PatchedChatOpenAI(model="safe-model", api_key="test-key")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="deerflow.models.patched_openai"):
|
||||
with pytest.raises(TimeoutError):
|
||||
_ = [chunk async for chunk in model._astream([])]
|
||||
|
||||
summaries = [record.getMessage() for record in caplog.records if "OPENAI_TOOL_STREAM_SUMMARY" in record.getMessage()]
|
||||
assert len(summaries) == 1
|
||||
assert "status=error" in summaries[0]
|
||||
assert "error_type=TimeoutError" in summaries[0]
|
||||
assert "TOP_SECRET_EXCEPTION_DETAIL" not in summaries[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_stream_snapshots_are_isolated_between_concurrent_invocations(monkeypatch):
|
||||
both_streams_arrived = asyncio.Event()
|
||||
arrived = 0
|
||||
|
||||
async def fake_parent_astream(self, *args, **kwargs):
|
||||
nonlocal arrived
|
||||
raw_arguments = kwargs["raw_arguments"]
|
||||
prefix = raw_arguments[:24]
|
||||
converted = self._convert_chunk_to_generation_chunk(_chunk(prefix), AIMessageChunk, None)
|
||||
assert converted is not None
|
||||
yield converted
|
||||
arrived += 1
|
||||
if arrived == 2:
|
||||
both_streams_arrived.set()
|
||||
await both_streams_arrived.wait()
|
||||
converted = self._convert_chunk_to_generation_chunk(_chunk(raw_arguments), AIMessageChunk, None)
|
||||
assert converted is not None
|
||||
yield converted
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", fake_parent_astream)
|
||||
model = PatchedChatOpenAI(model="safe-model", api_key="test-key")
|
||||
|
||||
async def collect_args(raw_arguments: str) -> str:
|
||||
pieces = [chunk.message.tool_call_chunks[0]["args"] async for chunk in model._astream([], raw_arguments=raw_arguments) if chunk.message.tool_call_chunks]
|
||||
return "".join(pieces)
|
||||
|
||||
first_arguments = '{"path":"/mnt/user-data/outputs/one.md","content":"first"}'
|
||||
second_arguments = '{"path":"/mnt/user-data/outputs/two.md","content":"second"}'
|
||||
first, second = await asyncio.gather(
|
||||
collect_args(first_arguments),
|
||||
collect_args(second_arguments),
|
||||
)
|
||||
|
||||
assert first == first_arguments
|
||||
assert second == second_arguments
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_stream_can_be_closed_from_caller_context(monkeypatch, caplog):
|
||||
async def fake_parent_astream(self, *args, **kwargs):
|
||||
converted = self._convert_chunk_to_generation_chunk(_chunk("{}"), AIMessageChunk, None)
|
||||
assert converted is not None
|
||||
yield converted
|
||||
await asyncio.Event().wait()
|
||||
|
||||
monkeypatch.setattr(ChatOpenAI, "_astream", fake_parent_astream)
|
||||
model = PatchedChatOpenAI(model="safe-model", api_key="test-key")
|
||||
stream = model._astream([])
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="deerflow.models.patched_openai"):
|
||||
first = await asyncio.create_task(anext(stream))
|
||||
assert first.message.tool_call_chunks
|
||||
await stream.aclose()
|
||||
|
||||
summaries = [record for record in caplog.records if "OPENAI_TOOL_STREAM_SUMMARY" in record.getMessage()]
|
||||
assert len(summaries) == 1
|
||||
assert summaries[0].levelno == logging.INFO
|
||||
assert "status=cancelled" in summaries[0].getMessage()
|
||||
+4743
File diff suppressed because it is too large.
Load diff
+186
@@ -0,0 +1,186 @@
|
||||
"""A completed/cancelled graph must not leave SSE alive on metadata I/O."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.runtime import journal as journal_module
|
||||
from deerflow.runtime.runs import worker
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
from deerflow.runtime.runs.schemas import RunStatus
|
||||
from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge
|
||||
|
||||
|
||||
class FinishedAgent:
|
||||
async def astream(self, *args, **kwargs):
|
||||
yield {"messages": []}
|
||||
|
||||
|
||||
async def make_run(monkeypatch, phase, *, graph=None, ignore_cancellation=False):
|
||||
monkeypatch.setattr(worker, "_RUN_FINALIZATION_TIMEOUT_SECONDS", 0.03)
|
||||
entered = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
drained = asyncio.Event()
|
||||
|
||||
async def blocked(*args, **kwargs):
|
||||
entered.set()
|
||||
try:
|
||||
await release.wait()
|
||||
except asyncio.CancelledError:
|
||||
cancelled.set()
|
||||
if ignore_cancellation:
|
||||
await release.wait()
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
drained.set()
|
||||
|
||||
fake_journal = SimpleNamespace(flush=AsyncMock(), get_completion_data=lambda: {})
|
||||
monkeypatch.setattr(journal_module, "RunJournal", lambda **kwargs: fake_journal)
|
||||
manager = RunManager()
|
||||
record = await manager.create("cleanup-test")
|
||||
bridge = MemoryStreamBridge()
|
||||
# Preserve the stream for assertions and avoid leaving a 60s sleeper.
|
||||
monkeypatch.setattr(bridge, "cleanup", AsyncMock())
|
||||
thread_store = SimpleNamespace(update_display_name=AsyncMock(), update_status=AsyncMock())
|
||||
checkpoint_calls = 0
|
||||
|
||||
async def checkpoint(*args):
|
||||
nonlocal checkpoint_calls
|
||||
checkpoint_calls += 1
|
||||
# Snapshot and terminal inspection precede final metadata cleanup.
|
||||
if (phase == "checkpoint" and checkpoint_calls == 3) or (phase == "terminal_checkpoint" and checkpoint_calls == 2):
|
||||
await blocked()
|
||||
return SimpleNamespace(checkpoint={"channel_values": {"title": "Test"}}, config={})
|
||||
|
||||
checkpointer = SimpleNamespace(aget_tuple=checkpoint)
|
||||
if phase == "journal":
|
||||
fake_journal.flush = blocked
|
||||
elif phase == "completion":
|
||||
monkeypatch.setattr(manager, "update_run_completion", blocked)
|
||||
elif phase == "title":
|
||||
thread_store.update_display_name = blocked
|
||||
elif phase == "thread_status":
|
||||
thread_store.update_status = blocked
|
||||
elif phase == "terminal_status":
|
||||
async def persist_status(run_id, status, **kwargs):
|
||||
if status != "running":
|
||||
await blocked()
|
||||
manager._store = SimpleNamespace(update_status=persist_status, update_run_completion=AsyncMock())
|
||||
task = asyncio.create_task(worker.run_agent(
|
||||
bridge, manager, record,
|
||||
ctx=worker.RunContext(checkpointer=checkpointer, event_store=object(), thread_store=thread_store),
|
||||
agent_factory=lambda **kwargs: graph or FinishedAgent(),
|
||||
graph_input={}, config={},
|
||||
))
|
||||
record.task = task
|
||||
return SimpleNamespace(
|
||||
task=task, manager=manager, record=record, bridge=bridge,
|
||||
entered=entered, cancelled=cancelled, release=release, drained=drained,
|
||||
journal=fake_journal, thread_store=thread_store,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("phase", ["journal", "completion", "checkpoint", "title", "thread_status", "terminal_status", "terminal_checkpoint"])
|
||||
async def test_metadata_stall_always_ends_sse(monkeypatch, phase, caplog):
|
||||
run = await make_run(monkeypatch, phase)
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
await asyncio.wait_for(run.cancelled.wait(), 1)
|
||||
assert run.entered.is_set()
|
||||
assert run.record.status == (RunStatus.error if phase == "terminal_checkpoint" else RunStatus.success)
|
||||
assert run.bridge._streams[run.record.run_id].ended
|
||||
assert "RUN_COMPLETION_TIMEOUT" in caplog.text
|
||||
if phase == "terminal_checkpoint":
|
||||
assert any(event.event == "error" for event in run.bridge._streams[run.record.run_id].events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delayed_cancellation_acknowledgement_does_not_hold_stream(monkeypatch):
|
||||
run = await make_run(monkeypatch, "journal", ignore_cancellation=True)
|
||||
try:
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
assert run.bridge._streams[run.record.run_id].ended
|
||||
await asyncio.wait_for(run.cancelled.wait(), 1)
|
||||
finally:
|
||||
run.release.set()
|
||||
await asyncio.wait_for(run.drained.wait(), 1)
|
||||
await asyncio.sleep(0)
|
||||
run.thread_store.update_display_name.assert_not_awaited()
|
||||
run.thread_store.update_status.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_graph_ends_sse_even_when_cleanup_stalls(monkeypatch):
|
||||
graph_started = asyncio.Event()
|
||||
|
||||
class WaitingAgent:
|
||||
async def astream(self, *args, **kwargs):
|
||||
graph_started.set()
|
||||
await asyncio.Event().wait()
|
||||
yield {}
|
||||
|
||||
run = await make_run(monkeypatch, "journal", graph=WaitingAgent())
|
||||
await asyncio.wait_for(graph_started.wait(), 1)
|
||||
await run.manager.cancel(run.record.run_id)
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
assert run.record.status == RunStatus.interrupted
|
||||
assert run.bridge._streams[run.record.run_id].ended
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_during_cleanup_preserves_terminator(monkeypatch):
|
||||
run = await make_run(monkeypatch, "journal")
|
||||
await asyncio.wait_for(run.entered.wait(), 1)
|
||||
run.task.cancel()
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
assert run.bridge._streams[run.record.run_id].ended
|
||||
await asyncio.wait_for(run.cancelled.wait(), 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_healthy_completion_still_persists_metadata(monkeypatch):
|
||||
run = await make_run(monkeypatch, "healthy")
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
run.journal.flush.assert_awaited_once()
|
||||
run.thread_store.update_display_name.assert_awaited_once_with("cleanup-test", "Test")
|
||||
run.thread_store.update_status.assert_awaited_once_with("cleanup-test", "idle")
|
||||
assert run.record.status == RunStatus.success
|
||||
assert run.bridge._streams[run.record.run_id].ended
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_older_cleanup_does_not_overwrite_newer_run_metadata(monkeypatch):
|
||||
run = await make_run(monkeypatch, "journal")
|
||||
await asyncio.wait_for(run.entered.wait(), 1)
|
||||
await run.manager.create(run.record.thread_id)
|
||||
run.release.set()
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
run.thread_store.update_display_name.assert_not_awaited()
|
||||
run.thread_store.update_status.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_budget_does_not_time_out_executing_graph(monkeypatch):
|
||||
graph_started = asyncio.Event()
|
||||
graph_release = asyncio.Event()
|
||||
|
||||
class LongAgent:
|
||||
async def astream(self, *args, **kwargs):
|
||||
graph_started.set()
|
||||
await graph_release.wait()
|
||||
yield {}
|
||||
|
||||
run = await make_run(monkeypatch, "healthy", graph=LongAgent())
|
||||
try:
|
||||
await asyncio.wait_for(graph_started.wait(), 1)
|
||||
done, _ = await asyncio.wait({run.task}, timeout=0.06)
|
||||
assert not done
|
||||
assert run.record.status == RunStatus.running
|
||||
assert not run.bridge._streams[run.record.run_id].ended
|
||||
finally:
|
||||
graph_release.set()
|
||||
await asyncio.wait_for(run.task, 1)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""SSE create responses must point the SDK at the resumable GET endpoint."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.routers import runs, thread_runs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stateless", [False, True])
|
||||
def test_create_stream_headers_keep_metadata_and_resume_paths_distinct(monkeypatch, stateless):
|
||||
module = runs if stateless else thread_runs
|
||||
record = SimpleNamespace(run_id="run-123")
|
||||
start_run = AsyncMock(return_value=record)
|
||||
monkeypatch.setattr(module, "start_run", start_run)
|
||||
monkeypatch.setattr(module, "get_stream_bridge", lambda request: object())
|
||||
monkeypatch.setattr(module, "get_run_manager", lambda request: object())
|
||||
if stateless:
|
||||
monkeypatch.setattr(module, "_resolve_thread_id", lambda body: "thread-123")
|
||||
|
||||
async def consumer(*args):
|
||||
yield "event: end\ndata: {}\n\n"
|
||||
|
||||
monkeypatch.setattr(module, "sse_consumer", consumer)
|
||||
request = Request({"type": "http", "app": FastAPI()})
|
||||
body = thread_runs.RunCreateRequest()
|
||||
if stateless:
|
||||
response = asyncio.run(runs.stateless_stream(body, request))
|
||||
else:
|
||||
response = asyncio.run(thread_runs.stream_run.__wrapped__("thread-123", body, request))
|
||||
assert response.status_code == 200 # Location is not an HTTP redirect.
|
||||
assert response.headers["Content-Location"] == "/api/threads/thread-123/runs/run-123"
|
||||
assert response.headers["Location"] == "/threads/thread-123/runs/run-123/stream"
|
||||
assert response.headers["X-Accel-Buffering"] == "no"
|
||||
assert response.headers["Cache-Control"] == "no-cache"
|
||||
assert response.media_type == "text/event-stream"
|
||||
start_run.assert_awaited_once()
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
|
||||
_SKILL_VIEW_KEYS = ("skill_view_reuse_enabled", "skill_view_cache_enabled")
|
||||
_OPT_IN_KEYS = ("model_timing_nonblocking_enabled", "asyncio_thread_pool_enabled")
|
||||
|
||||
|
||||
def test_missing_runtime_performance_enables_skill_view_only():
|
||||
config = AppConfig(sandbox={"use": "deerflow.sandbox.local:LocalSandboxProvider"})
|
||||
assert config.runtime_performance.model_dump() == {
|
||||
"skill_view_reuse_enabled": True,
|
||||
"skill_view_cache_enabled": True,
|
||||
"model_timing_nonblocking_enabled": False,
|
||||
"asyncio_thread_pool_enabled": False,
|
||||
"asyncio_thread_pool_size": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [*_SKILL_VIEW_KEYS, *_OPT_IN_KEYS])
|
||||
def test_runtime_performance_switches_are_independent(tmp_path: Path, key: str):
|
||||
# Skill-view switches default on, so independence is checked by turning
|
||||
# one of them off. The other two switches default off and are turned on.
|
||||
enabled = key not in _SKILL_VIEW_KEYS
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
f"sandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\nruntime_performance:\n {key}: {'true' if enabled else 'false'}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
flags = AppConfig.from_file(str(config_path)).runtime_performance.model_dump()
|
||||
assert flags[key] is enabled
|
||||
for other in _SKILL_VIEW_KEYS:
|
||||
if other != key:
|
||||
assert flags[other] is True
|
||||
for other in _OPT_IN_KEYS:
|
||||
if other != key:
|
||||
assert flags[other] is False
|
||||
|
||||
|
||||
def test_unavailable_config_does_not_enable_optimizations(monkeypatch):
|
||||
from deerflow.config import app_config
|
||||
from deerflow.config.runtime_performance_config import get_runtime_performance_config
|
||||
|
||||
def unavailable():
|
||||
raise FileNotFoundError("not initialized")
|
||||
|
||||
monkeypatch.setattr(app_config, "get_app_config", unavailable)
|
||||
flags = get_runtime_performance_config().model_dump()
|
||||
assert not any(value for key, value in flags.items() if key.endswith("_enabled"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [0, -1, 1025, True, 1.5, "100"])
|
||||
def test_config_rejects_invalid_thread_pool_sizes(size):
|
||||
from pydantic import ValidationError
|
||||
|
||||
from deerflow.config.runtime_performance_config import RuntimePerformanceConfig
|
||||
|
||||
with pytest.raises(ValidationError, match="asyncio_thread_pool_size"):
|
||||
RuntimePerformanceConfig(asyncio_thread_pool_size=size)
|
||||
|
||||
|
||||
def test_config_accepts_thread_pool_size_100_without_enabling_it():
|
||||
from deerflow.config.runtime_performance_config import RuntimePerformanceConfig
|
||||
|
||||
config = RuntimePerformanceConfig(asyncio_thread_pool_size=100)
|
||||
assert config.asyncio_thread_pool_size == 100
|
||||
assert config.asyncio_thread_pool_enabled is False
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
"""Commands must return after timeout even when descendants inherit output."""
|
||||
|
||||
import ctypes
|
||||
import io
|
||||
import os
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from deerflow.community.aio_sandbox import aio_sandbox as aio_module
|
||||
from deerflow.config.sandbox_config import SandboxConfig
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def python_sandbox(monkeypatch):
|
||||
sandbox = LocalSandbox("command-liveness-test")
|
||||
# A Python parent stands in for a shell, keeping these process-tree tests
|
||||
# independent of bash installation and PowerShell quoting differences.
|
||||
monkeypatch.setattr(sandbox, "_get_shell", lambda: sys.executable)
|
||||
monkeypatch.setattr(sandbox, "_command_timeout_seconds", lambda: 10)
|
||||
return sandbox
|
||||
|
||||
|
||||
def _is_running(pid: int) -> bool:
|
||||
if os.name == "nt":
|
||||
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
kernel32.OpenProcess.restype = ctypes.c_void_p
|
||||
kernel32.OpenProcess.argtypes = [ctypes.c_uint, ctypes.c_bool, ctypes.c_uint]
|
||||
kernel32.WaitForSingleObject.argtypes = [ctypes.c_void_p, ctypes.c_uint]
|
||||
kernel32.CloseHandle.argtypes = [ctypes.c_void_p]
|
||||
handle = kernel32.OpenProcess(0x00100000, False, pid) # SYNCHRONIZE
|
||||
if not handle:
|
||||
return False
|
||||
try:
|
||||
return kernel32.WaitForSingleObject(handle, 0) == 0x102 # WAIT_TIMEOUT
|
||||
finally:
|
||||
kernel32.CloseHandle(handle)
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
# A killed grandchild can briefly remain as a zombie until its new parent
|
||||
# reaps it. It cannot execute work or retain pipe handles in this state.
|
||||
stat = Path(f"/proc/{pid}/stat")
|
||||
if stat.exists() and stat.read_text().rsplit(")", 1)[1].split()[0] == "Z":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def test_command_preserves_stdout_stderr_and_exit_code(python_sandbox):
|
||||
output = python_sandbox.execute_command("import sys; print('stdout marker'); print('stderr marker', file=sys.stderr); sys.exit(7)")
|
||||
assert "stdout marker" in output
|
||||
assert "Std Error:\nstderr marker" in output
|
||||
assert "Exit Code: 7" in output
|
||||
|
||||
|
||||
def test_command_stdin_is_closed(python_sandbox):
|
||||
output = python_sandbox.execute_command("import sys; print(repr(sys.stdin.read()))")
|
||||
assert output.strip() == "''"
|
||||
|
||||
|
||||
def test_command_timeout_terminates_child_that_inherits_output(python_sandbox, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(python_sandbox, "_command_timeout_seconds", lambda: 2)
|
||||
pid_file = tmp_path / "child.pid"
|
||||
child = "import time; time.sleep(60)"
|
||||
command = f"import pathlib,subprocess,sys,time; child=subprocess.Popen([sys.executable,'-c',{child!r}]); pathlib.Path({str(pid_file)!r}).write_text(str(child.pid)); print('started', flush=True); time.sleep(60)"
|
||||
started = time.monotonic()
|
||||
child_pid = None
|
||||
try:
|
||||
output = python_sandbox.execute_command(command)
|
||||
assert time.monotonic() - started < 8
|
||||
assert output.startswith("Error: Command timed out after 2 seconds")
|
||||
assert "started" in output
|
||||
child_pid = int(pid_file.read_text())
|
||||
deadline = time.monotonic() + 2
|
||||
while _is_running(child_pid) and time.monotonic() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert not _is_running(child_pid), "Timed-out skill child was left running"
|
||||
finally:
|
||||
if child_pid is None and pid_file.exists():
|
||||
child_pid = int(pid_file.read_text())
|
||||
if child_pid is not None and _is_running(child_pid):
|
||||
os.kill(child_pid, signal.SIGTERM)
|
||||
|
||||
|
||||
def test_shell_exit_does_not_wait_for_descendants_output_handles(python_sandbox, tmp_path):
|
||||
pid_file = tmp_path / "background-child.pid"
|
||||
command = f"import pathlib,subprocess,sys; child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(60)']); pathlib.Path({str(pid_file)!r}).write_text(str(child.pid)); print('parent done')"
|
||||
try:
|
||||
started = time.monotonic()
|
||||
output = python_sandbox.execute_command(command)
|
||||
assert time.monotonic() - started < 8
|
||||
assert output.strip() == "parent done"
|
||||
finally:
|
||||
if pid_file.exists():
|
||||
child_pid = int(pid_file.read_text())
|
||||
if _is_running(child_pid):
|
||||
os.kill(child_pid, signal.SIGTERM)
|
||||
|
||||
|
||||
def test_large_command_output_is_bounded_before_tool_truncation(python_sandbox):
|
||||
output = python_sandbox.execute_command("import sys; sys.stdout.write('HEAD' + 'x' * (3 * 1024 * 1024) + 'TAIL')")
|
||||
assert output.startswith("HEAD") and output.endswith("TAIL")
|
||||
assert "command output truncated" in output
|
||||
assert len(output) < LocalSandbox._COMMAND_OUTPUT_MAX_BYTES + 100
|
||||
|
||||
|
||||
def test_runaway_output_is_stopped_before_command_timeout(python_sandbox, monkeypatch):
|
||||
monkeypatch.setattr(python_sandbox, "_COMMAND_CAPTURE_MAX_BYTES", 8192)
|
||||
output = python_sandbox.execute_command("import sys,time\nwhile True:\n sys.stdout.write('x' * 4096); sys.stdout.flush(); time.sleep(0.01)")
|
||||
assert output.startswith("Error: Command output exceeded 8192 bytes")
|
||||
|
||||
|
||||
def test_command_timeout_setting_and_default(python_sandbox, monkeypatch):
|
||||
from deerflow import config
|
||||
|
||||
settings = SandboxConfig(use="test", command_timeout_seconds=12.5)
|
||||
monkeypatch.setattr(config, "get_app_config", lambda: SimpleNamespace(sandbox=settings))
|
||||
assert LocalSandbox._command_timeout_seconds(python_sandbox) == 12.5
|
||||
assert SandboxConfig(use="test").command_timeout_seconds == 600
|
||||
for invalid in (0, -1, float("inf"), float("nan")):
|
||||
with pytest.raises(ValidationError):
|
||||
SandboxConfig(use="test", command_timeout_seconds=invalid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["read_file", "read_file_range"])
|
||||
def test_local_file_read_rejects_pipe_before_reading(monkeypatch, operation):
|
||||
read_fd, write_fd = os.pipe()
|
||||
try:
|
||||
monkeypatch.setattr(os, "open", lambda *_args: os.dup(read_fd))
|
||||
with pytest.raises(OSError, match="Only regular files"):
|
||||
getattr(LocalSandbox("test"), operation)("/virtual/fifo")
|
||||
finally:
|
||||
os.close(read_fd)
|
||||
os.close(write_fd)
|
||||
|
||||
|
||||
def test_local_range_reader_stops_at_budget_without_draining_long_line(monkeypatch):
|
||||
class LongLine(io.StringIO):
|
||||
reads = 0
|
||||
|
||||
def readline(self, size=-1):
|
||||
self.reads += 1
|
||||
if self.reads > 1:
|
||||
raise AssertionError("Read the rest of the long line after filling the result")
|
||||
return "x" * 4097
|
||||
|
||||
sandbox = LocalSandbox("test")
|
||||
monkeypatch.setattr(sandbox, "_open_regular_text", lambda *_args, **_kwargs: LongLine())
|
||||
assert sandbox.read_file_range("/virtual/long-line", max_chars=100) == ("x" * 100, 2, True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("operation", "args"),
|
||||
[
|
||||
("execute_command", ("echo test",)),
|
||||
("read_file_range", ("/tmp/skill.md",)),
|
||||
("list_dir", ("/tmp",)),
|
||||
("write_file", ("/tmp/output", "test")),
|
||||
("update_file", ("/tmp/output", b"test")),
|
||||
],
|
||||
)
|
||||
def test_aio_busy_session_fails_without_executing_or_releasing_other_request(monkeypatch, operation, args):
|
||||
client = Mock()
|
||||
monkeypatch.setattr(aio_module, "AioSandboxClient", Mock(return_value=client))
|
||||
sandbox = aio_module.AioSandbox("test", "http://127.0.0.1:1")
|
||||
sandbox._LOCK_WAIT_TIMEOUT = 0.01
|
||||
sandbox._lock.acquire()
|
||||
try:
|
||||
started = time.monotonic()
|
||||
with pytest.raises(TimeoutError, match="waiting for the previous operation"):
|
||||
getattr(sandbox, operation)(*args)
|
||||
assert time.monotonic() - started < 1
|
||||
assert sandbox._lock.locked()
|
||||
assert client.mock_calls == []
|
||||
finally:
|
||||
sandbox._lock.release()
|
||||
client.shell.exec_command.return_value = SimpleNamespace(data=SimpleNamespace(output="next command"))
|
||||
assert sandbox.execute_command("echo next") == "next command"
|
||||
assert not sandbox._lock.locked()
|
||||
|
||||
|
||||
def test_aio_range_reader_preserves_continuation_and_rejects_non_files(monkeypatch, tmp_path):
|
||||
def execute_script(*, command, no_change_timeout):
|
||||
# Run only the generated bounded-reader program against test fixtures;
|
||||
# no container or external sandbox service is contacted.
|
||||
args = shlex.split(command)
|
||||
completed = subprocess.run([sys.executable, *args[1:]], capture_output=True, encoding="utf-8", timeout=5)
|
||||
return SimpleNamespace(data=SimpleNamespace(output=completed.stdout + completed.stderr))
|
||||
|
||||
client = Mock()
|
||||
client.shell.exec_command.side_effect = execute_script
|
||||
monkeypatch.setattr(aio_module, "AioSandboxClient", Mock(return_value=client))
|
||||
sandbox = aio_module.AioSandbox("test", "http://127.0.0.1:1")
|
||||
source = tmp_path / "long.txt"
|
||||
source.write_text("x" * 10_000 + "\nsecond\nthird", encoding="utf-8")
|
||||
assert sandbox.read_file_range(str(source), max_chars=100) == ("x" * 100, 2, True)
|
||||
assert sandbox.read_file_range(str(source), start_line=2, end_line=2, max_chars=100) == ("second", None, False)
|
||||
with pytest.raises(OSError, match="bounded sandbox read failed"):
|
||||
sandbox.read_file_range(str(tmp_path))
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Regression coverage for the opt-in online-skill metadata cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config import paths as paths_module
|
||||
from deerflow.sandbox import skill_view
|
||||
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
|
||||
|
||||
|
||||
def _skill(root: Path, directory: str, name: str | None = None) -> Path:
|
||||
location = root / "public" / directory
|
||||
location.mkdir(parents=True, exist_ok=True)
|
||||
(location / "SKILL.md").write_text(f"---\nname: {name or directory}\ndescription: cache test\n---\nSkill body\n", encoding="utf-8")
|
||||
return location
|
||||
|
||||
|
||||
def _wait_for_warm_validation_followers(expected: int, timeout: float = 5) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
with skill_view._WARM_VALIDATION_LOCK:
|
||||
if any(flight.followers >= expected for flight in skill_view._WARM_VALIDATION_FLIGHTS.values()):
|
||||
return
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"warm validation did not collect {expected} followers")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
root = tmp_path / "skills"
|
||||
_skill(root, "online")
|
||||
config = tmp_path / "extensions.json"
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": True}}}), encoding="utf-8")
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config))
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "runtime"))
|
||||
monkeypatch.setattr(paths_module, "_paths", None)
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: True)
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
yield root, config, LocalSkillStorage(host_path=str(root))
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
|
||||
|
||||
def test_warm_cache_does_not_walk_parse_copy_or_take_refresh_lock(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
|
||||
def unexpected(*args, **kwargs):
|
||||
raise AssertionError("warm cache executed an expensive refresh operation")
|
||||
|
||||
monkeypatch.setattr(storage, "load_skills", unexpected)
|
||||
monkeypatch.setattr(skill_view, "_scan_tree", unexpected)
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", unexpected)
|
||||
monkeypatch.setattr(skill_view, "_view_lock", unexpected)
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is first
|
||||
assert first.enabled_roots == (("online", "public", "online"),)
|
||||
|
||||
|
||||
def test_offline_and_invalid_config_invalidate_immediately(cache_setup):
|
||||
root, config, storage = cache_setup
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}}}), encoding="utf-8")
|
||||
offline = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not offline.enabled_names
|
||||
assert not (offline.root / "public" / "online").exists()
|
||||
assert (root / "public" / "online" / "SKILL.md").exists()
|
||||
config.write_text("{", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="not valid JSON"):
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not (first.root / skill_view.SKILL_VIEW_MARKER).exists()
|
||||
assert not list(first.root.iterdir())
|
||||
assert not skill_view._VIEW_CACHE
|
||||
|
||||
|
||||
def test_external_source_change_same_size_preserved_mtime_and_new_files(cache_setup):
|
||||
root, _config, storage = cache_setup
|
||||
source = root / "public" / "online" / "helper.py"
|
||||
source.write_text("VALUE=1\n", encoding="utf-8")
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
before = source.stat()
|
||||
source.write_text("VALUE=2\n", encoding="utf-8")
|
||||
os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
updated = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert updated.generation != first.generation
|
||||
assert (updated.root / "public" / "online" / "helper.py").read_text() == "VALUE=2\n"
|
||||
(source.parent / "new-reference.md").write_text("fresh", encoding="utf-8")
|
||||
new_file_view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert (new_file_view.root / "public" / "online" / "new-reference.md").read_text() == "fresh"
|
||||
source.unlink()
|
||||
deleted = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not (deleted.root / "public" / "online" / "helper.py").exists()
|
||||
|
||||
|
||||
def test_nested_new_skill_and_support_manifest_are_detected(cache_setup):
|
||||
root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_skill(root, "nested/new", "new-skill")
|
||||
support = root / "public" / "shared-lib"
|
||||
support.mkdir()
|
||||
(support / "helper.py").write_text("value", encoding="utf-8")
|
||||
manifest = support / skill_view.SKILL_SUPPORT_MANIFEST
|
||||
manifest.write_text(json.dumps({"required_by": ["online"]}), encoding="utf-8")
|
||||
changed = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert ("new-skill", "public", "nested/new") in changed.enabled_roots
|
||||
assert (changed.root / "public" / "shared-lib" / "helper.py").exists()
|
||||
manifest.write_text(json.dumps({"required_by": ["not-online"]}), encoding="utf-8")
|
||||
removed = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not (removed.root / "public" / "shared-lib").exists()
|
||||
|
||||
|
||||
def test_destination_tampering_is_repaired_even_with_preserved_size_mtime(cache_setup):
|
||||
root, _config, storage = cache_setup
|
||||
source = root / "public" / "online" / "helper.py"
|
||||
source.write_text("VALUE=1\n", encoding="utf-8")
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
destination = view.root / "public" / "online" / "helper.py"
|
||||
before = destination.stat()
|
||||
destination.write_text("VALUE=9\n", encoding="utf-8")
|
||||
os.utime(destination, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
repaired = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert (repaired.root / "public" / "online" / "helper.py").read_text() == "VALUE=1\n"
|
||||
rogue = view.root / "public" / "unknown"
|
||||
rogue.mkdir()
|
||||
(rogue / "secret").write_text("unexpected", encoding="utf-8")
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not rogue.exists()
|
||||
|
||||
|
||||
def test_second_process_cache_and_force_rebuild(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
marker_metadata = (first.root / skill_view.SKILL_VIEW_MARKER).stat()
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
original_load = storage.load_skills
|
||||
count = 0
|
||||
|
||||
def counted(**kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
return original_load(**kwargs)
|
||||
|
||||
monkeypatch.setattr(storage, "load_skills", counted)
|
||||
second = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert second == first
|
||||
assert count == 0
|
||||
assert (first.root / skill_view.SKILL_VIEW_MARKER).stat().st_mtime_ns == marker_metadata.st_mtime_ns
|
||||
child = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"""
|
||||
import sys
|
||||
from deerflow.sandbox import skill_view
|
||||
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
|
||||
skill_view._skill_view_cache_enabled = lambda: True
|
||||
storage = LocalSkillStorage(host_path=sys.argv[1])
|
||||
def unexpected(**kwargs):
|
||||
raise AssertionError('another process unnecessarily rediscovered skills')
|
||||
storage.load_skills = unexpected
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
print(view.generation)
|
||||
""",
|
||||
str(_root),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=True,
|
||||
)
|
||||
assert child.stdout.strip() == first.generation
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage, force=True)
|
||||
assert count == 1
|
||||
|
||||
|
||||
def test_concurrent_cold_calls_join_one_refresh(cache_setup, monkeypatch, caplog):
|
||||
_root, _config, storage = cache_setup
|
||||
original = storage.load_skills
|
||||
count = 0
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def counted(**kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
return original(**kwargs)
|
||||
|
||||
def call():
|
||||
barrier.wait(timeout=5)
|
||||
return skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
|
||||
monkeypatch.setattr(storage, "load_skills", counted)
|
||||
with caplog.at_level(logging.INFO, logger=skill_view.__name__), concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: call(), range(8)))
|
||||
assert count == 1
|
||||
assert len({result.generation for result in results}) == 1
|
||||
assert "lock_wait_ms=" in caplog.text
|
||||
assert "refresh_ms=" in caplog.text
|
||||
assert "source_entries=" in caplog.text
|
||||
assert "Skill body" not in caplog.text
|
||||
|
||||
|
||||
def test_concurrent_warm_calls_share_only_the_overlapping_validation(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
original = skill_view._entry_matches
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
count = 0
|
||||
count_lock = threading.Lock()
|
||||
|
||||
def blocked(entry, skills_root, config_key):
|
||||
nonlocal count
|
||||
with count_lock:
|
||||
count += 1
|
||||
entered.set()
|
||||
assert release.wait(timeout=5)
|
||||
return original(entry, skills_root, config_key)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", blocked)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(timeout=5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(7)]
|
||||
_wait_for_warm_validation_followers(7)
|
||||
release.set()
|
||||
results = [leader.result(timeout=5), *(future.result(timeout=5) for future in followers)]
|
||||
|
||||
assert count == 1
|
||||
assert all(result is first for result in results)
|
||||
assert not skill_view._WARM_VALIDATION_FLIGHTS
|
||||
|
||||
# There is no TTL/result cache: a caller after the completed batch validates again.
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is first
|
||||
assert count == 2
|
||||
|
||||
|
||||
def test_warm_validation_failure_reaches_followers_and_next_call_recovers(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
original = skill_view._entry_matches
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
count = 0
|
||||
|
||||
def broken(*_args):
|
||||
nonlocal count
|
||||
count += 1
|
||||
entered.set()
|
||||
assert release.wait(timeout=5)
|
||||
raise RuntimeError("simulated warm validation failure")
|
||||
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", broken)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(timeout=5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(5)]
|
||||
_wait_for_warm_validation_followers(5)
|
||||
release.set()
|
||||
for future in [leader, *followers]:
|
||||
with pytest.raises(RuntimeError, match="simulated warm validation failure"):
|
||||
future.result(timeout=5)
|
||||
|
||||
assert count == 1
|
||||
assert not skill_view._WARM_VALIDATION_FLIGHTS
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", original)
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is first
|
||||
|
||||
|
||||
def test_warm_validation_with_new_config_hash_does_not_join_old_flight(cache_setup, monkeypatch):
|
||||
_root, config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
original = skill_view._entry_matches
|
||||
old_key = skill_view._config_key()
|
||||
old_entered = threading.Event()
|
||||
release_old = threading.Event()
|
||||
observed_keys: list[str] = []
|
||||
observed_lock = threading.Lock()
|
||||
|
||||
def block_old_config(entry, skills_root, config_key):
|
||||
with observed_lock:
|
||||
observed_keys.append(config_key)
|
||||
if config_key == old_key:
|
||||
old_entered.set()
|
||||
assert release_old.wait(timeout=5)
|
||||
return original(entry, skills_root, config_key)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", block_old_config)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
old_call = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert old_entered.wait(timeout=5)
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}}}), encoding="utf-8")
|
||||
new_key = skill_view._config_key()
|
||||
assert new_key != old_key
|
||||
new_call = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
try:
|
||||
# This must finish while the old-hash leader is still blocked.
|
||||
changed = new_call.result(timeout=5)
|
||||
finally:
|
||||
release_old.set()
|
||||
joined_after_change = old_call.result(timeout=5)
|
||||
|
||||
assert not changed.enabled_names
|
||||
assert not joined_after_change.enabled_names
|
||||
assert observed_keys.count(old_key) == 1
|
||||
assert new_key in observed_keys
|
||||
assert not skill_view._WARM_VALIDATION_FLIGHTS
|
||||
|
||||
|
||||
def test_disabled_switch_retains_full_refresh_and_legacy_marker(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: False)
|
||||
original = storage.load_skills
|
||||
count = 0
|
||||
|
||||
def counted(**kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
return original(**kwargs)
|
||||
|
||||
monkeypatch.setattr(storage, "load_skills", counted)
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert count == 2
|
||||
assert not first.generation
|
||||
assert "generation" not in json.loads(first.marker_content)
|
||||
assert first.enabled_roots == (("online", "public", "online"),)
|
||||
|
||||
|
||||
def test_custom_storage_instances_do_not_share_filesystem_cache(cache_setup):
|
||||
root, _config, _storage = cache_setup
|
||||
_skill(root, "other")
|
||||
|
||||
class FilteredStorage(LocalSkillStorage):
|
||||
def __init__(self, selected):
|
||||
super().__init__(host_path=str(root))
|
||||
self.selected = selected
|
||||
|
||||
def load_skills(self, **kwargs):
|
||||
return [skill for skill in super().load_skills(**kwargs) if skill.name == self.selected]
|
||||
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=FilteredStorage("online"))
|
||||
second = skill_view.refresh_skill_sandbox_view(storage=FilteredStorage("other"))
|
||||
assert first.enabled_names == {"online"}
|
||||
assert second.enabled_names == {"other"}
|
||||
assert not first.generation and not second.generation
|
||||
|
||||
|
||||
def test_config_change_mid_refresh_is_not_published(cache_setup, monkeypatch):
|
||||
_root, config, storage = cache_setup
|
||||
original = skill_view._copy_file_atomic
|
||||
changed = False
|
||||
|
||||
def copy_and_disable(source, destination):
|
||||
nonlocal changed
|
||||
original(source, destination)
|
||||
if not changed:
|
||||
changed = True
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}}}), encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", copy_and_disable)
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not view.enabled_names
|
||||
assert not (view.root / "public" / "online").exists()
|
||||
|
||||
|
||||
def test_switching_cache_off_and_back_on_does_not_reuse_a_stale_marker(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
original = storage.load_skills
|
||||
count = 0
|
||||
|
||||
def counted(**kwargs):
|
||||
nonlocal count
|
||||
count += 1
|
||||
return original(**kwargs)
|
||||
|
||||
monkeypatch.setattr(storage, "load_skills", counted)
|
||||
first = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: False)
|
||||
(root / "public" / "online" / "reference.md").write_text("new content", encoding="utf-8")
|
||||
legacy = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert count == 3
|
||||
assert not legacy.generation
|
||||
assert "generation" not in json.loads(legacy.marker_content)
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: True)
|
||||
restored = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert restored.generation != first.generation
|
||||
assert (restored.root / "public" / "online" / "reference.md").read_text() == "new content"
|
||||
assert count == 4
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is restored
|
||||
assert count == 4
|
||||
|
||||
|
||||
def test_marker_damage_and_copy_failure_clear_capability(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
marker = view.root / skill_view.SKILL_VIEW_MARKER
|
||||
marker.write_text("not-a-valid-marker", encoding="utf-8")
|
||||
repaired = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert marker.read_text() == repaired.marker_content
|
||||
(root / "public" / "online" / "SKILL.md").write_text("---\nname: online\ndescription: changed\n---\n", encoding="utf-8")
|
||||
|
||||
def broken_copy(*args):
|
||||
raise OSError("simulated disk failure")
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", broken_copy)
|
||||
with pytest.raises(OSError, match="simulated disk failure"):
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not marker.exists()
|
||||
assert not skill_view._VIEW_CACHE
|
||||
assert not list(view.root.iterdir())
|
||||
|
||||
|
||||
def test_symlink_destination_does_not_modify_external_directory(cache_setup, tmp_path):
|
||||
_root, _config, storage = cache_setup
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
external = tmp_path / "outside"
|
||||
external.mkdir()
|
||||
(external / "keep").write_text("untouched", encoding="utf-8")
|
||||
target = view.root / "public" / "online"
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(target)
|
||||
try:
|
||||
target.symlink_to(external, target_is_directory=True)
|
||||
except OSError:
|
||||
if os.name != "nt":
|
||||
pytest.skip("directory symlinks are unavailable on this platform")
|
||||
junction = subprocess.run(["cmd", "/c", "mklink", "/J", str(target), str(external)], capture_output=True, timeout=10)
|
||||
if junction.returncode:
|
||||
pytest.skip("directory links are unavailable on this platform")
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not skill_view._is_view_link(target)
|
||||
assert (external / "keep").read_text() == "untouched"
|
||||
assert not (external / "SKILL.md").exists()
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
"""The admin timing page explains whether a wait is the skill check or the command."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from deerflow.sandbox.skill_view import (
|
||||
_EVENT_LOCK,
|
||||
_SKILL_VIEW_EVENTS,
|
||||
begin_skill_view_trace,
|
||||
commit_skill_view_trace,
|
||||
diagnose_skill_view,
|
||||
end_skill_view_trace,
|
||||
explain_skill_view_event,
|
||||
note_skill_view_check,
|
||||
query_skill_view_events,
|
||||
recent_skill_view_events,
|
||||
record_skill_view_event,
|
||||
remember_invoked_skill,
|
||||
reset_skill_view_events,
|
||||
)
|
||||
|
||||
|
||||
def _event(status: str, total_ms: float, **extra) -> dict:
|
||||
return {
|
||||
"status": status,
|
||||
"total_ms": total_ms,
|
||||
"lock_wait_ms": extra.get("lock_wait_ms", 0),
|
||||
"source_entries": extra.get("source_entries", 0),
|
||||
"path_count": extra.get("path_count", 0),
|
||||
}
|
||||
|
||||
|
||||
def test_switch_card_is_omitted():
|
||||
report = diagnose_skill_view(cache_enabled=False, reuse_enabled=False, events=[])
|
||||
assert all(item["id"] != "switches" for item in report["checks"])
|
||||
|
||||
|
||||
def test_fast_skip_points_at_the_command_itself():
|
||||
report = diagnose_skill_view(
|
||||
cache_enabled=True,
|
||||
reuse_enabled=True,
|
||||
events=[_event("skipped", 12), _event("narrow", 40, path_count=1)],
|
||||
)
|
||||
assert report["level"] == "ok"
|
||||
latest = next(item for item in report["checks"] if item["id"] == "latest")
|
||||
assert "命令" in latest["look_at"] or "脚本" in latest["cause"]
|
||||
repeat = next(item for item in report["checks"] if item["id"] == "repeat")
|
||||
assert repeat["level"] == "ok"
|
||||
|
||||
|
||||
def test_repeated_full_refresh_is_called_out():
|
||||
events = [_event("refreshed", 18000, source_entries=4000), _event("hit", 9000, source_entries=4000)]
|
||||
report = diagnose_skill_view(cache_enabled=True, reuse_enabled=True, events=events)
|
||||
assert report["level"] == "bad"
|
||||
assert "4000" in report["conclusion"]
|
||||
repeat = next(item for item in report["checks"] if item["id"] == "repeat")
|
||||
assert "2 条" in repeat["result"]
|
||||
|
||||
|
||||
def test_lock_wait_is_separated_from_a_file_scan():
|
||||
explained = explain_skill_view_event(_event("joined", 12000, lock_wait_ms=11000))
|
||||
assert explained["level"] == "warn"
|
||||
assert "等" in explained["headline"]
|
||||
|
||||
|
||||
def test_command_and_skill_are_saved_only_after_the_call_returns():
|
||||
reset_skill_view_events()
|
||||
try:
|
||||
note_skill_view_check(status="narrow", total_ms=30, path_count=1)
|
||||
remember_invoked_skill("task-report-build")
|
||||
assert recent_skill_view_events() == []
|
||||
token = begin_skill_view_trace()
|
||||
try:
|
||||
note_skill_view_check(status="narrow", total_ms=30, path_count=1)
|
||||
remember_invoked_skill("task-report-build")
|
||||
assert recent_skill_view_events() == []
|
||||
commit_skill_view_trace(
|
||||
tool="bash",
|
||||
command="python /mnt/skills/public/task-report-build/scripts/run.py\necho done",
|
||||
skill_names=["directory-name"],
|
||||
tool_ms=8000,
|
||||
allow_empty=True,
|
||||
)
|
||||
finally:
|
||||
end_skill_view_trace(token)
|
||||
event = recent_skill_view_events()[0]
|
||||
assert event["status"] == "narrow"
|
||||
assert event["tool"] == "bash"
|
||||
assert event["command"] == "python /mnt/skills/public/task-report-build/scripts/run.py echo done"
|
||||
assert event["skill_names"] == ["task-report-build"]
|
||||
assert event["tool_ms"] == 8000
|
||||
report = diagnose_skill_view(cache_enabled=True, reuse_enabled=True)
|
||||
latest = next(item for item in report["checks"] if item["id"] == "latest")
|
||||
assert latest["skill_names"] == ["task-report-build"]
|
||||
assert "task-report-build" in report["events"][0]["explanation"]["headline"]
|
||||
finally:
|
||||
reset_skill_view_events()
|
||||
|
||||
|
||||
def test_diagnostic_write_cannot_escape_into_the_tool(monkeypatch):
|
||||
def boom(**kwargs):
|
||||
raise RuntimeError("diagnostic store failed")
|
||||
|
||||
monkeypatch.setattr("deerflow.sandbox.skill_view.record_skill_view_event", boom)
|
||||
token = begin_skill_view_trace()
|
||||
try:
|
||||
note_skill_view_check(status="skipped")
|
||||
commit_skill_view_trace(tool="bash", command="echo hello", allow_empty=True)
|
||||
finally:
|
||||
end_skill_view_trace(token)
|
||||
|
||||
|
||||
def test_event_buffer_keeps_the_newest_sample_first():
|
||||
reset_skill_view_events()
|
||||
try:
|
||||
record_skill_view_event(status="skipped", total_ms=5)
|
||||
record_skill_view_event(status="narrow", total_ms=20, path_count=1)
|
||||
report = diagnose_skill_view(cache_enabled=True, reuse_enabled=True)
|
||||
assert report["events"][0]["status"] == "narrow"
|
||||
assert report["events"][1]["status"] == "skipped"
|
||||
assert report["events"][0]["explanation"]["level"] == "ok"
|
||||
finally:
|
||||
reset_skill_view_events()
|
||||
|
||||
|
||||
def test_events_older_than_one_day_are_dropped():
|
||||
reset_skill_view_events()
|
||||
try:
|
||||
record_skill_view_event(status="narrow", total_ms=10, tool="bash", command="echo now", tool_ms=100)
|
||||
record_skill_view_event(status="skipped", total_ms=5, tool="bash", command="echo old", tool_ms=9000)
|
||||
old = (datetime.now(UTC) - timedelta(days=2)).isoformat(timespec="seconds")
|
||||
with _EVENT_LOCK:
|
||||
_SKILL_VIEW_EVENTS[1]["at"] = old
|
||||
page = query_skill_view_events(page=1, page_size=20, sort="time", order="desc")
|
||||
assert page["total"] == 1
|
||||
assert [item["command"] for item in page["events"]] == ["echo now"]
|
||||
finally:
|
||||
reset_skill_view_events()
|
||||
|
||||
|
||||
def test_events_can_be_paged_and_sorted_by_time_or_duration():
|
||||
reset_skill_view_events()
|
||||
try:
|
||||
record_skill_view_event(status="skipped", total_ms=10, tool="bash", command="first", tool_ms=100)
|
||||
record_skill_view_event(status="narrow", total_ms=4000, tool="bash", command="second", tool_ms=50)
|
||||
record_skill_view_event(status="skipped", total_ms=20, tool="read_file", command="third", tool_ms=8000)
|
||||
by_duration = query_skill_view_events(page=1, page_size=2, sort="duration", order="desc")
|
||||
assert by_duration["total"] == 3
|
||||
assert by_duration["page"] == 1
|
||||
assert [item["command"] for item in by_duration["events"]] == ["third", "first"]
|
||||
by_check = query_skill_view_events(page=1, page_size=3, sort="check", order="desc")
|
||||
assert [item["command"] for item in by_check["events"]] == ["second", "third", "first"]
|
||||
by_time = query_skill_view_events(page=2, page_size=2, sort="time", order="asc")
|
||||
assert by_time["page"] == 2
|
||||
assert [item["command"] for item in by_time["events"]] == ["third"]
|
||||
finally:
|
||||
reset_skill_view_events()
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Tool hot path: skip untouched skills, and sync one skill after a management write."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config import paths as paths_module
|
||||
from deerflow.sandbox import skill_view, tools
|
||||
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
|
||||
|
||||
|
||||
def _skill(root: Path, directory: str, name: str | None = None, body: str = "Skill body") -> Path:
|
||||
location = root / "public" / directory
|
||||
location.mkdir(parents=True, exist_ok=True)
|
||||
(location / "SKILL.md").write_text(
|
||||
f"---\nname: {name or directory}\ndescription: fast path test\n---\n{body}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return location
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
root = tmp_path / "skills"
|
||||
_skill(root, "online")
|
||||
_skill(root, "other")
|
||||
config = tmp_path / "extensions.json"
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": True}, "other": {"enabled": True}}}), encoding="utf-8")
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config))
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "runtime"))
|
||||
monkeypatch.setattr(paths_module, "_paths", None)
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: True)
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
storage = LocalSkillStorage(host_path=str(root))
|
||||
monkeypatch.setattr("deerflow.skills.storage.get_or_new_skill_storage", lambda **_kwargs: storage)
|
||||
yield root, config, storage
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
|
||||
|
||||
def _forbid_full_scan(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def fail(*_args, **_kwargs):
|
||||
raise AssertionError("full skill-tree scan")
|
||||
|
||||
monkeypatch.setattr(skill_view, "_scan_tree", fail)
|
||||
|
||||
def fail_load(self, *args, **kwargs):
|
||||
raise AssertionError("full skill catalogue load")
|
||||
|
||||
monkeypatch.setattr(LocalSkillStorage, "load_skills", fail_load)
|
||||
|
||||
|
||||
def test_file_sync_updates_only_that_skill_and_stays_warm(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
other_before = (view.root / "public" / "other" / "SKILL.md").read_bytes()
|
||||
(root / "public" / "online" / "SKILL.md").write_text(
|
||||
"---\nname: online\ndescription: fast path test\n---\nupdated body\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_forbid_full_scan(monkeypatch)
|
||||
synced = skill_view.sync_skill_sandbox_skill("online", storage=storage)
|
||||
|
||||
assert synced is not None
|
||||
assert (synced.root / "public" / "online" / "SKILL.md").read_text(encoding="utf-8").endswith("updated body\n")
|
||||
assert (synced.root / "public" / "other" / "SKILL.md").read_bytes() == other_before
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage).enabled_names == {"online", "other"}
|
||||
|
||||
|
||||
def test_disable_removes_one_skill_without_a_full_scan(cache_setup, monkeypatch):
|
||||
root, config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}, "other": {"enabled": True}}}), encoding="utf-8")
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
synced = skill_view.sync_skill_sandbox_skill("online", storage=storage)
|
||||
|
||||
assert synced is not None
|
||||
assert not (synced.root / "public" / "online").exists()
|
||||
assert (root / "public" / "online" / "SKILL.md").is_file()
|
||||
assert (synced.root / "public" / "other" / "SKILL.md").is_file()
|
||||
assert "online" not in synced.enabled_names
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage).enabled_names == {"other"}
|
||||
|
||||
|
||||
def test_delete_removes_the_copy_when_the_source_is_gone(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(root / "public" / "other")
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
synced = skill_view.sync_skill_sandbox_skill("other", storage=storage)
|
||||
|
||||
assert synced is not None
|
||||
assert not (synced.root / "public" / "other").exists()
|
||||
assert "other" not in synced.enabled_names
|
||||
assert (synced.root / "public" / "online" / "SKILL.md").is_file()
|
||||
|
||||
|
||||
def test_invoked_skill_does_not_scan_the_others(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
(root / "public" / "online" / "helper.py").write_text("value = 1\n", encoding="utf-8")
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
(root / "public" / "online" / "helper.py").write_text("value = 2\n", encoding="utf-8")
|
||||
walked: list[str] = []
|
||||
real_add = skill_view._add_tree
|
||||
|
||||
def tracking(source, *args, **kwargs):
|
||||
walked.append(Path(source).name)
|
||||
return real_add(source, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_add_tree", tracking)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
view = skill_view.prepare_skill_tool_view(["/mnt/skills/public/online/helper.py"], storage=storage)
|
||||
|
||||
assert walked == ["online"]
|
||||
assert view is not None
|
||||
assert (view.root / "public" / "online" / "helper.py").read_text(encoding="utf-8") == "value = 2\n"
|
||||
assert "other" not in walked
|
||||
|
||||
|
||||
def test_command_without_a_skill_does_not_refresh(cache_setup, monkeypatch):
|
||||
_root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(skill_view, "refresh_skill_sandbox_view", lambda **_kwargs: calls.append("refresh"))
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
assert skill_view.prepare_skill_tool_view([], storage=storage) is not None
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_offline_path_is_removed_and_not_enabled(cache_setup, monkeypatch):
|
||||
root, config, storage = cache_setup
|
||||
_skill(root, "offline")
|
||||
config.write_text(
|
||||
json.dumps({"skills": {"online": {"enabled": True}, "other": {"enabled": True}, "offline": {"enabled": False}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
assert not (skill_view.peek_skill_sandbox_view(storage=storage).root / "public" / "offline").exists()
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
view = skill_view.prepare_skill_tool_view(["/mnt/skills/public/offline/SKILL.md"], storage=storage)
|
||||
|
||||
assert view is not None
|
||||
assert "offline" not in view.enabled_names
|
||||
assert not (view.root / "public" / "offline").exists()
|
||||
assert (root / "public" / "offline" / "SKILL.md").is_file()
|
||||
|
||||
|
||||
def test_new_skill_is_copied_when_first_invoked(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_skill(root, "fresh", body="fresh body")
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
view = skill_view.prepare_skill_tool_view(["/mnt/skills/public/fresh/SKILL.md"], storage=storage)
|
||||
|
||||
assert view is not None
|
||||
assert ("fresh", "public", "fresh") in view.enabled_roots
|
||||
assert "fresh body" in (view.root / "public" / "fresh" / "SKILL.md").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _forbid_catalogue_walk(root: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
catalogue = {root / "public", root / "custom"}
|
||||
real_walk = skill_view.os.walk
|
||||
|
||||
def guarded(start, *args, **kwargs):
|
||||
if Path(start) in catalogue:
|
||||
raise AssertionError(f"walked the skill catalogue at {start}")
|
||||
return real_walk(start, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(skill_view.os, "walk", guarded)
|
||||
|
||||
|
||||
def test_sync_uses_the_canonical_directory(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_skill(root, "fresh", body="fresh body")
|
||||
nested = root / "public" / "online" / "examples"
|
||||
nested.mkdir()
|
||||
(nested / "SKILL.md").write_text("---\nname: nested\ndescription: decoy\n---\nnope\n", encoding="utf-8")
|
||||
_forbid_full_scan(monkeypatch)
|
||||
_forbid_catalogue_walk(root, monkeypatch)
|
||||
|
||||
synced = skill_view.sync_skill_sandbox_skill("fresh", storage=storage)
|
||||
|
||||
assert synced is not None
|
||||
assert "fresh body" in (synced.root / "public" / "fresh" / "SKILL.md").read_text(encoding="utf-8")
|
||||
assert (synced.root / "public" / "other" / "SKILL.md").is_file()
|
||||
|
||||
|
||||
def test_misplaced_skill_still_resolves_by_frontmatter(cache_setup, monkeypatch):
|
||||
root, _config, storage = cache_setup
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
_skill(root, "misplaced", name="wanted", body="wanted body")
|
||||
walked: list[Path] = []
|
||||
real_walk = skill_view.os.walk
|
||||
|
||||
def tracking(start, *args, **kwargs):
|
||||
walked.append(Path(start))
|
||||
return real_walk(start, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(skill_view.os, "walk", tracking)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
synced = skill_view.sync_skill_sandbox_skill("wanted", storage=storage)
|
||||
|
||||
assert root / "public" in walked
|
||||
assert synced is not None
|
||||
assert "wanted body" in (synced.root / "public" / "misplaced" / "SKILL.md").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_support_library_follows_the_skill_it_serves(cache_setup, monkeypatch):
|
||||
root, config, storage = cache_setup
|
||||
support = root / "public" / "shared-lib"
|
||||
support.mkdir()
|
||||
(support / "helper.py").write_text("shared = 1\n", encoding="utf-8")
|
||||
(support / skill_view.SKILL_SUPPORT_MANIFEST).write_text(json.dumps({"required_by": ["online"]}), encoding="utf-8")
|
||||
skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
(support / "helper.py").write_text("shared = 2\n", encoding="utf-8")
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
view = skill_view.sync_skill_sandbox_skill("online", storage=storage)
|
||||
|
||||
assert view is not None
|
||||
assert (view.root / "public" / "shared-lib" / "helper.py").read_text(encoding="utf-8") == "shared = 2\n"
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}, "other": {"enabled": True}}}), encoding="utf-8")
|
||||
disabled = skill_view.sync_skill_sandbox_skill("online", storage=storage)
|
||||
assert disabled is not None
|
||||
assert not (disabled.root / "public" / "shared-lib").exists()
|
||||
assert (disabled.root / "public" / "other" / "SKILL.md").is_file()
|
||||
|
||||
|
||||
def _tool_sandbox(tmp_path: Path, view_root: Path, monkeypatch: pytest.MonkeyPatch) -> Mock:
|
||||
sandbox = Mock()
|
||||
sandbox.execute_command.return_value = "ok"
|
||||
sandbox.read_file_range.return_value = ("skill body", None, False)
|
||||
monkeypatch.setattr(tools, "ensure_sandbox_initialized", lambda _runtime: sandbox)
|
||||
monkeypatch.setattr(tools, "ensure_thread_directories_exist", lambda _runtime: None)
|
||||
monkeypatch.setattr(tools, "is_local_sandbox", lambda _runtime: True)
|
||||
monkeypatch.setattr(tools, "is_host_bash_allowed", lambda: True)
|
||||
monkeypatch.setattr(tools, "get_thread_data", lambda _runtime: {"workspace_path": str(tmp_path / "workspace"), "uploads_path": str(tmp_path / "uploads"), "outputs_path": str(tmp_path / "outputs")})
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda _path: None)
|
||||
monkeypatch.setattr(tools, "_get_skills_container_path", lambda: "/mnt/skills")
|
||||
(tmp_path / "workspace").mkdir(exist_ok=True)
|
||||
assert view_root.is_dir()
|
||||
return sandbox
|
||||
|
||||
|
||||
def test_bash_without_a_skill_does_not_scan(cache_setup, monkeypatch, tmp_path):
|
||||
_root, _config, storage = cache_setup
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
sandbox = _tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
result = tools.bash_tool.func(runtime=None, description="test", command="echo hello")
|
||||
|
||||
assert result == "ok"
|
||||
sandbox.execute_command.assert_called_once()
|
||||
assert "/mnt/skills" not in sandbox.execute_command.call_args.args[0]
|
||||
|
||||
|
||||
def test_bash_runs_the_updated_skill_script(cache_setup, monkeypatch, tmp_path):
|
||||
root, _config, storage = cache_setup
|
||||
script = root / "public" / "online" / "scripts"
|
||||
script.mkdir()
|
||||
(script / "run.py").write_text("print('one')\n", encoding="utf-8")
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
(script / "run.py").write_text("print('two')\n", encoding="utf-8")
|
||||
sandbox = _tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
result = tools.bash_tool.func(runtime=None, description="test", command="python /mnt/skills/public/online/scripts/run.py")
|
||||
|
||||
assert result == "ok"
|
||||
command = sandbox.execute_command.call_args.args[0]
|
||||
assert "public/online/scripts/run.py" in command.replace("\\", "/")
|
||||
assert (view.root / "public" / "online" / "scripts" / "run.py").read_text(encoding="utf-8") == "print('two')\n"
|
||||
|
||||
|
||||
def test_bash_rejects_an_offline_skill(cache_setup, monkeypatch, tmp_path):
|
||||
root, config, storage = cache_setup
|
||||
_skill(root, "offline")
|
||||
config.write_text(
|
||||
json.dumps({"skills": {"online": {"enabled": True}, "other": {"enabled": True}, "offline": {"enabled": False}}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
sandbox = _tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
result = tools.bash_tool.func(runtime=None, description="test", command="python /mnt/skills/public/offline/SKILL.md")
|
||||
|
||||
assert result.startswith("Error:")
|
||||
sandbox.execute_command.assert_not_called()
|
||||
|
||||
|
||||
def test_read_file_uses_the_synced_copy(cache_setup, monkeypatch, tmp_path):
|
||||
root, _config, storage = cache_setup
|
||||
view = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
(root / "public" / "online" / "SKILL.md").write_text(
|
||||
"---\nname: online\ndescription: fast path test\n---\nfrom disk\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
skill_view.sync_skill_sandbox_skill("online", storage=storage)
|
||||
sandbox = _tool_sandbox(tmp_path, view.root, monkeypatch)
|
||||
_forbid_full_scan(monkeypatch)
|
||||
|
||||
result = tools.read_file_tool.func(runtime=None, description="test", path="/mnt/skills/public/online/SKILL.md")
|
||||
|
||||
assert result == "skill body"
|
||||
read_path = sandbox.read_file_range.call_args.args[0]
|
||||
assert Path(read_path).is_relative_to(view.root)
|
||||
assert "from disk" in Path(read_path).read_text(encoding="utf-8")
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
"""Concurrent cold/invalidated refreshes share the published verification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config import paths as paths_module
|
||||
from deerflow.sandbox import skill_view
|
||||
from deerflow.skills.storage.local_skill_storage import LocalSkillStorage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def refresh_setup(tmp_path, monkeypatch):
|
||||
root = tmp_path / "skills"
|
||||
directory = root / "public" / "online"
|
||||
directory.mkdir(parents=True)
|
||||
(directory / "SKILL.md").write_text("---\nname: online\ndescription: refresh test\n---\nBody\n", encoding="utf-8")
|
||||
config = tmp_path / "extensions.json"
|
||||
config.write_text('{"skills":{}}', encoding="utf-8")
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config))
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "runtime"))
|
||||
monkeypatch.setattr(paths_module, "_paths", None)
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: True)
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
yield root, config, LocalSkillStorage(host_path=str(root))
|
||||
skill_view.reset_skill_sandbox_view_cache()
|
||||
|
||||
|
||||
def wait_for_followers(count):
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
with skill_view._REFRESH_FLIGHTS_LOCK:
|
||||
if any(flight.followers == count for flight in skill_view._REFRESH_FLIGHTS.values()):
|
||||
return
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"Expected {count} overlapping refresh followers")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_overlapping_refreshes_verify_published_tree_once(refresh_setup, monkeypatch, invalidated):
|
||||
root, _config, storage = refresh_setup
|
||||
old = skill_view.refresh_skill_sandbox_view(storage=storage) if invalidated else None
|
||||
(root / "public" / "online" / "asset.txt").write_text("fresh", encoding="utf-8")
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
original_matches = skill_view._entry_matches
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
published_checks = 0
|
||||
|
||||
def blocked_copy(source, destination):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_copy(source, destination)
|
||||
|
||||
def counted(entry, *args):
|
||||
nonlocal published_checks
|
||||
if old is None or entry.view.generation != old.generation:
|
||||
published_checks += 1
|
||||
return original_matches(entry, *args)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", blocked_copy)
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", counted)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(7)]
|
||||
try:
|
||||
wait_for_followers(7)
|
||||
finally:
|
||||
release.set()
|
||||
views = [future.result(timeout=5) for future in [leader, *followers]]
|
||||
assert all(view is views[0] for view in views)
|
||||
assert published_checks == 1
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is views[0]
|
||||
assert published_checks == 2 # no completed-result cache or TTL
|
||||
|
||||
|
||||
def test_refresh_failure_reaches_followers_and_clears_old_capability(refresh_setup, monkeypatch):
|
||||
root, _config, storage = refresh_setup
|
||||
old = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
(root / "public" / "online" / "asset.txt").write_text("changed", encoding="utf-8")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
calls = 0
|
||||
|
||||
def broken(*args):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
raise OSError("simulated refresh disk failure")
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", broken)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(7)]
|
||||
try:
|
||||
wait_for_followers(7)
|
||||
finally:
|
||||
release.set()
|
||||
for future in [leader, *followers]:
|
||||
with pytest.raises(OSError, match="simulated refresh disk failure"):
|
||||
future.result(timeout=5)
|
||||
assert calls == 1
|
||||
assert not skill_view._VIEW_CACHE
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
assert not (old.root / skill_view.SKILL_VIEW_MARKER).exists()
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", original_copy)
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage).enabled_names == {"online"}
|
||||
|
||||
|
||||
def test_disable_during_refresh_never_reaches_any_follower_as_enabled(refresh_setup, monkeypatch):
|
||||
_root, config, storage = refresh_setup
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
|
||||
def blocked_copy(source, destination):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_copy(source, destination)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", blocked_copy)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(7)]
|
||||
try:
|
||||
wait_for_followers(7)
|
||||
config.write_text(json.dumps({"skills": {"online": {"enabled": False}}}), encoding="utf-8")
|
||||
finally:
|
||||
release.set()
|
||||
views = [future.result(timeout=5) for future in [leader, *followers]]
|
||||
assert all(not view.enabled_names for view in views)
|
||||
assert not (views[0].root / "public" / "online").exists()
|
||||
|
||||
|
||||
def test_cache_off_overlapping_refreshes_copy_once(refresh_setup, monkeypatch):
|
||||
"""Parallel skill reads must not each queue on the process lock."""
|
||||
_root, _config, storage = refresh_setup
|
||||
monkeypatch.setattr(skill_view, "_skill_view_cache_enabled", lambda: False)
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
copies = 0
|
||||
|
||||
def blocked_copy(source, destination):
|
||||
nonlocal copies
|
||||
copies += 1
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_copy(source, destination)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", blocked_copy)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
followers = [pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage) for _ in range(3)]
|
||||
try:
|
||||
wait_for_followers(3)
|
||||
finally:
|
||||
release.set()
|
||||
views = [future.result(timeout=5) for future in [leader, *followers]]
|
||||
assert copies == 1
|
||||
assert all(view.enabled_names == {"online"} for view in views)
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
|
||||
|
||||
def test_refresh_follower_timeout_keeps_leader_and_recovers(refresh_setup, monkeypatch):
|
||||
_root, _config, storage = refresh_setup
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
monkeypatch.setattr(skill_view, "_REBUILD_WAIT_SECONDS", 0.05)
|
||||
|
||||
def blocked_copy(source, destination):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_copy(source, destination)
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", blocked_copy)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
follower = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
try:
|
||||
with pytest.raises(TimeoutError, match="skill view refresh"):
|
||||
follower.result(timeout=2)
|
||||
assert len(skill_view._REFRESH_FLIGHTS) == 1
|
||||
assert not leader.done()
|
||||
finally:
|
||||
release.set()
|
||||
view = leader.result(timeout=5)
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
assert skill_view.refresh_skill_sandbox_view(storage=storage) is view
|
||||
|
||||
|
||||
def test_new_availability_hash_cannot_join_old_refresh(monkeypatch):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
key = ("source", "runtime", "storage")
|
||||
old_view, new_view = object(), object()
|
||||
|
||||
def old_refresh():
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return old_view
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
old = pool.submit(skill_view._refresh_singleflight, key, "old-config", old_refresh)
|
||||
assert entered.wait(5)
|
||||
try:
|
||||
new = pool.submit(skill_view._refresh_singleflight, key, "new-config", lambda: new_view)
|
||||
assert new.result(timeout=2) is new_view
|
||||
assert not old.done()
|
||||
finally:
|
||||
release.set()
|
||||
assert old.result(timeout=5) is old_view
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("revocation", ["config", "marker"])
|
||||
def test_follower_rejects_capability_revoked_after_publication(refresh_setup, monkeypatch, revocation):
|
||||
_root, config, storage = refresh_setup
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original_copy = skill_view._copy_file_atomic
|
||||
|
||||
def blocked_copy(source, destination):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_copy(source, destination)
|
||||
|
||||
class RevocationEvent(threading.Event):
|
||||
def wait(self, timeout=None):
|
||||
completed = super().wait(timeout)
|
||||
if completed:
|
||||
if revocation == "config":
|
||||
config.write_text('{"skills":{"online":{"enabled":false}}}', encoding="utf-8")
|
||||
else:
|
||||
marker = next(iter(skill_view._VIEW_CACHE.values())).view.root / skill_view.SKILL_VIEW_MARKER
|
||||
marker.write_text("revoked", encoding="utf-8")
|
||||
return completed
|
||||
|
||||
monkeypatch.setattr(skill_view, "_copy_file_atomic", blocked_copy)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
leader = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
assert entered.wait(5)
|
||||
with skill_view._REFRESH_FLIGHTS_LOCK:
|
||||
next(iter(skill_view._REFRESH_FLIGHTS.values())).done = RevocationEvent()
|
||||
follower = pool.submit(skill_view.refresh_skill_sandbox_view, storage=storage)
|
||||
try:
|
||||
wait_for_followers(1)
|
||||
finally:
|
||||
release.set()
|
||||
leader.result(timeout=5)
|
||||
with pytest.raises(RuntimeError, match="changed while waiting"):
|
||||
follower.result(timeout=5)
|
||||
assert not skill_view._REFRESH_FLIGHTS
|
||||
recovered = skill_view.refresh_skill_sandbox_view(storage=storage)
|
||||
if revocation == "config":
|
||||
assert not recovered.enabled_names
|
||||
else:
|
||||
assert (recovered.root / skill_view.SKILL_VIEW_MARKER).read_text() == recovered.marker_content
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.config.runtime_performance_config import RuntimePerformanceConfig
|
||||
from deerflow.sandbox import skill_view, skill_view_scope, tools
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tool_env(tmp_path: Path, monkeypatch):
|
||||
from deerflow.config import app_config
|
||||
|
||||
options = RuntimePerformanceConfig(skill_view_reuse_enabled=True, skill_view_cache_enabled=False)
|
||||
monkeypatch.setattr(app_config, "get_app_config", lambda: SimpleNamespace(runtime_performance=options))
|
||||
view = SimpleNamespace(root=tmp_path / "view", enabled_roots=(("real-name", "public", "nested/demo"),))
|
||||
refresh = Mock(return_value=view)
|
||||
monkeypatch.setattr(skill_view, "_refresh_uncached_skill_sandbox_view", refresh)
|
||||
sandbox = Mock()
|
||||
sandbox.read_file_range.return_value = ("skill body", None, False)
|
||||
sandbox.list_dir.return_value = [str(view.root / "public/nested/demo/SKILL.md")]
|
||||
sandbox.glob.return_value = ([str(view.root / "public/nested/demo/SKILL.md")], False)
|
||||
sandbox.grep.return_value = ([tools.GrepMatch(path=str(view.root / "public/nested/demo/SKILL.md"), line_number=1, line="body")], False)
|
||||
sandbox.execute_command.return_value = str(view.root / "public/nested/demo/SKILL.md")
|
||||
|
||||
def initialize(_runtime):
|
||||
# Providers also ask for the capability during acquisition/attestation.
|
||||
skill_view.refresh_skill_sandbox_view()
|
||||
return sandbox
|
||||
|
||||
thread_data = {key: str(tmp_path / key) for key in ("workspace_path", "uploads_path", "outputs_path")}
|
||||
monkeypatch.setattr(tools, "ensure_sandbox_initialized", initialize)
|
||||
monkeypatch.setattr(tools, "ensure_thread_directories_exist", lambda _: None)
|
||||
monkeypatch.setattr(tools, "is_local_sandbox", lambda _: True)
|
||||
monkeypatch.setattr(tools, "is_host_bash_allowed", lambda: True)
|
||||
monkeypatch.setattr(tools, "get_thread_data", lambda _: thread_data)
|
||||
monkeypatch.setattr(tools, "_record_skill_read", lambda _: None)
|
||||
monkeypatch.setattr(tools, "_get_skills_container_path", lambda: "/mnt/skills")
|
||||
monkeypatch.setattr(tools, "_get_acp_workspace_host_path", lambda _: None)
|
||||
monkeypatch.setattr(tools, "_get_custom_mounts", lambda: [])
|
||||
return refresh, sandbox, view
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["read_file", "ls", "glob", "grep", "bash"])
|
||||
def test_each_tool_reuses_one_view_including_provider_and_output_masking(tool_env, name):
|
||||
refresh, sandbox, _ = tool_env
|
||||
kwargs = {"runtime": None, "description": "test"}
|
||||
if name == "bash":
|
||||
kwargs["command"] = "cat /mnt/skills/public/nested/demo/SKILL.md /mnt/skills/public/nested/demo/SKILL.md"
|
||||
else:
|
||||
kwargs["path"] = "/mnt/skills/public/nested/demo/SKILL.md" if name == "read_file" else "/mnt/skills/public/nested/demo"
|
||||
if name in {"glob", "grep"}:
|
||||
kwargs["pattern"] = "*" if name == "glob" else "body"
|
||||
result = getattr(tools, f"{name}_tool").func(**kwargs)
|
||||
assert not result.startswith("Error:"), result
|
||||
assert refresh.call_count == 1
|
||||
assert skill_view_scope.get_active_skill_view_scope() is None
|
||||
|
||||
|
||||
def test_next_tool_call_revalidates_and_denies_newly_offline_skill(tool_env):
|
||||
refresh, sandbox, view = tool_env
|
||||
kwargs = {"runtime": None, "description": "test", "path": "/mnt/skills/public/nested/demo/SKILL.md"}
|
||||
assert tools.read_file_tool.func(**kwargs) == "skill body"
|
||||
refresh.return_value = SimpleNamespace(root=view.root, enabled_roots=())
|
||||
assert tools.read_file_tool.func(**kwargs).startswith("Error: Permission denied")
|
||||
assert refresh.call_count == 2
|
||||
assert sandbox.read_file_range.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/mnt/skills", "/mnt/skills/public", "/mnt/skills/public/missing/SKILL.md", "/mnt/skills/public/nested/demo/../other"])
|
||||
def test_reused_view_still_denies_unknown_broad_and_traversal_paths(tool_env, path):
|
||||
_, sandbox, _ = tool_env
|
||||
assert tools.read_file_tool.func(runtime=None, description="test", path=path).startswith("Error: Permission denied")
|
||||
sandbox.read_file_range.assert_not_called()
|
||||
|
||||
|
||||
def test_failed_refresh_is_not_retried_by_error_masking(tool_env):
|
||||
refresh, sandbox, _ = tool_env
|
||||
refresh.side_effect = RuntimeError("invalid availability")
|
||||
result = tools.read_file_tool.func(runtime=None, description="test", path="/mnt/skills/public/nested/demo/SKILL.md")
|
||||
assert result.startswith("Error:")
|
||||
assert refresh.call_count == 1
|
||||
sandbox.read_file_range.assert_not_called()
|
||||
assert skill_view_scope.get_active_skill_view_scope() is None
|
||||
|
||||
|
||||
def test_scope_is_thread_local_and_restored_after_failure(tool_env):
|
||||
@skill_view_scope.with_skill_view_scope
|
||||
def snapshot(fail=False):
|
||||
current = skill_view_scope.get_active_skill_view_scope()
|
||||
if fail:
|
||||
raise ValueError("test")
|
||||
return current
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
scopes = list(pool.map(lambda _: snapshot(), range(4)))
|
||||
assert len({id(scope) for scope in scopes}) == 4
|
||||
with pytest.raises(ValueError):
|
||||
snapshot(True)
|
||||
assert skill_view_scope.get_active_skill_view_scope() is None
|
||||
|
||||
|
||||
def test_disabled_scope_preserves_repeated_refresh(tool_env, monkeypatch):
|
||||
monkeypatch.setattr(skill_view_scope, "get_runtime_performance_config", lambda: RuntimePerformanceConfig(skill_view_reuse_enabled=False))
|
||||
refresh, _, _ = tool_env
|
||||
|
||||
@skill_view_scope.with_skill_view_scope
|
||||
def repeat():
|
||||
skill_view.refresh_skill_sandbox_view()
|
||||
skill_view.refresh_skill_sandbox_view()
|
||||
|
||||
repeat()
|
||||
assert refresh.call_count == 2
|
||||
@@ -0,0 +1,109 @@
|
||||
"""A slow skill refresh must not keep all later tool workers waiting forever."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import errno
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.sandbox import skill_view
|
||||
|
||||
|
||||
def test_process_lock_wait_times_out_and_recovers(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(skill_view, "_REBUILD_WAIT_SECONDS", 0.03, raising=False)
|
||||
lock = threading.Lock()
|
||||
monkeypatch.setattr(skill_view, "_PROCESS_LOCK", lock)
|
||||
lock.acquire()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(lambda: skill_view._view_lock(tmp_path).__enter__())
|
||||
try:
|
||||
with pytest.raises(TimeoutError, match="skill.*lock"):
|
||||
future.result(timeout=1)
|
||||
finally:
|
||||
lock.release()
|
||||
finally:
|
||||
if lock.locked():
|
||||
lock.release()
|
||||
with skill_view._view_lock(tmp_path):
|
||||
assert lock.locked()
|
||||
assert not lock.locked()
|
||||
|
||||
|
||||
def test_warm_follower_timeout_does_not_replace_running_leader(monkeypatch):
|
||||
monkeypatch.setattr(skill_view, "_VIEW_WAIT_SECONDS", 0.03, raising=False)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
entry = object()
|
||||
key = ("source", "target", "scope")
|
||||
flights = {}
|
||||
monkeypatch.setattr(skill_view, "_WARM_VALIDATION_FLIGHTS", flights)
|
||||
|
||||
def validate(*args):
|
||||
entered.set()
|
||||
assert release.wait(2)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(skill_view, "_entry_matches", validate)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
leader = pool.submit(skill_view._entry_matches_singleflight, key, entry, Path("."), "config")
|
||||
assert entered.wait(1)
|
||||
follower = pool.submit(skill_view._entry_matches_singleflight, key, entry, Path("."), "config")
|
||||
try:
|
||||
with pytest.raises(TimeoutError, match="skill.*validation"):
|
||||
follower.result(timeout=1)
|
||||
assert len(flights) == 1
|
||||
assert not leader.done()
|
||||
finally:
|
||||
release.set()
|
||||
assert leader.result(timeout=1) is True
|
||||
assert not flights
|
||||
assert skill_view._entry_matches_singleflight(key, entry, Path("."), "config") is True
|
||||
|
||||
|
||||
def test_file_lock_contention_is_bounded_and_process_lock_released(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(skill_view, "_REBUILD_WAIT_SECONDS", 0.03, raising=False)
|
||||
|
||||
def busy(*args):
|
||||
raise OSError(errno.EAGAIN, "lock held by another worker")
|
||||
|
||||
if skill_view.os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
monkeypatch.setattr(msvcrt, "locking", busy)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
monkeypatch.setattr(fcntl, "flock", busy)
|
||||
with pytest.raises(TimeoutError, match="skill.*lock"):
|
||||
with skill_view._view_lock(tmp_path):
|
||||
pytest.fail("must not enter the protected view")
|
||||
assert not skill_view._process_lock_held()
|
||||
|
||||
|
||||
def test_same_thread_reenters_view_lock_without_waiting(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(skill_view, "_REBUILD_WAIT_SECONDS", 0.03, raising=False)
|
||||
with skill_view._view_lock(tmp_path):
|
||||
with skill_view._view_lock(tmp_path):
|
||||
assert getattr(skill_view._VIEW_LOCK_DEPTH, "depth", 0) == 2
|
||||
assert getattr(skill_view._VIEW_LOCK_DEPTH, "depth", 0) == 0
|
||||
|
||||
|
||||
def test_file_lock_non_contention_error_is_not_retried(tmp_path, monkeypatch):
|
||||
def broken(*args):
|
||||
raise OSError(errno.EIO, "disk failure")
|
||||
|
||||
if skill_view.os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
monkeypatch.setattr(msvcrt, "locking", broken)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
monkeypatch.setattr(fcntl, "flock", broken)
|
||||
with pytest.raises(OSError, match="disk failure"):
|
||||
with skill_view._view_lock(tmp_path):
|
||||
pytest.fail("must not enter the protected view")
|
||||
assert not skill_view._process_lock_held()
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
|
||||
from deerflow.runtime.task_analysis.thread_metadata import task_analysis_thread_metadata
|
||||
|
||||
|
||||
def test_task_analysis_thread_metadata_links_task_and_run_and_is_hidden_from_chats():
|
||||
metadata = task_analysis_thread_metadata(
|
||||
{"id": "task-108", "source_task_id": "cop-108"},
|
||||
run_id="run-1",
|
||||
role="seat",
|
||||
seat_id="researcher",
|
||||
)
|
||||
|
||||
assert metadata == {
|
||||
"thread_type": "task-analysis",
|
||||
"system": True,
|
||||
"task_analysis_task_id": "task-108",
|
||||
"task_analysis_source_task_id": "cop-108",
|
||||
"task_analysis_run_id": "run-1",
|
||||
"task_analysis_role": "seat",
|
||||
"task_analysis_seat": "researcher",
|
||||
}
|
||||
assert is_excluded_system_thread(metadata) is True
|
||||
|
||||
|
||||
def test_task_analysis_markers_hide_a_legacy_runtime_thread_without_system_flag():
|
||||
assert is_excluded_system_thread({"task_analysis_task_id": "task-108"}) is True
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
"""Dedicated taskId + goPath conversation-history coverage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.task_deeplink_threads.memory import (
|
||||
MemoryTaskDeeplinkThreadStore,
|
||||
)
|
||||
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
|
||||
from deerflow.persistence.task_deeplink_threads.sql import TaskDeeplinkThreadRepository
|
||||
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_meta.sql import ThreadMetaRepository
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedicated_history_is_partitioned_by_original_task_id_and_go_path():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(ThreadMetaRow.__table__.create)
|
||||
await connection.run_sync(TaskDeeplinkThreadRow.__table__.create)
|
||||
|
||||
thread_repo = ThreadMetaRepository(session_factory)
|
||||
history_repo = TaskDeeplinkThreadRepository(session_factory)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
await thread_repo.create(
|
||||
"rwfx-oldest",
|
||||
user_id="creator-a",
|
||||
display_name="任务分析一",
|
||||
metadata={
|
||||
"taskId": "108",
|
||||
"taskRecordId": "108",
|
||||
"taskGoPath": "rwfx",
|
||||
"thread_type": "task_deeplink",
|
||||
"agent_id": "rwfx-agent",
|
||||
},
|
||||
)
|
||||
await thread_repo.create(
|
||||
"three-q",
|
||||
user_id="creator-b",
|
||||
metadata={
|
||||
"taskId": "108",
|
||||
"taskRecordId": "108",
|
||||
"taskGoPath": "3qfx",
|
||||
"thread_type": "task_deeplink",
|
||||
"agent_id": "three-q-agent",
|
||||
},
|
||||
)
|
||||
await thread_repo.create(
|
||||
"xdfx-action",
|
||||
user_id="creator-c",
|
||||
metadata={
|
||||
"taskId": "108",
|
||||
"taskRecordId": "action-9",
|
||||
"taskGoPath": "xdfx",
|
||||
"thread_type": "task_deeplink",
|
||||
"actionId": "action-9",
|
||||
"agent_id": "xdfx-agent",
|
||||
},
|
||||
)
|
||||
# A legacy task thread has no taskGoPath marker and must stay out of
|
||||
# the new table: old data is not migrated implicitly.
|
||||
await thread_repo.create(
|
||||
"legacy-task-thread",
|
||||
user_id="creator-d",
|
||||
metadata={"taskId": "108", "agent_id": "rwfx-agent"},
|
||||
)
|
||||
|
||||
# Listing orders by the canonical thread metadata timestamp/title,
|
||||
# not stale duplicated presentation data in the dedicated index.
|
||||
async with session_factory() as session:
|
||||
row = await session.get(ThreadMetaRow, "rwfx-oldest")
|
||||
assert row is not None
|
||||
row.updated_at = now + timedelta(minutes=1)
|
||||
await session.commit()
|
||||
|
||||
rwfx = await history_repo.list_threads("108", "rwfx")
|
||||
three_q = await history_repo.list_threads("108", "3qfx")
|
||||
xdfx = await history_repo.list_threads("action-9", "xdfx")
|
||||
|
||||
assert [row["thread_id"] for row in rwfx] == ["rwfx-oldest"]
|
||||
assert rwfx[0]["values"] == {"title": "任务分析一"}
|
||||
assert [row["thread_id"] for row in three_q] == ["three-q"]
|
||||
assert [row["thread_id"] for row in xdfx] == ["xdfx-action"]
|
||||
|
||||
async with session_factory() as session:
|
||||
indexed_ids = set((await session.scalars(select(TaskDeeplinkThreadRow.thread_id))).all())
|
||||
assert indexed_ids == {"rwfx-oldest", "three-q", "xdfx-action"}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_registration_and_thread_delete_keep_index_in_sync():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(ThreadMetaRow.__table__.create)
|
||||
await connection.run_sync(TaskDeeplinkThreadRow.__table__.create)
|
||||
|
||||
thread_repo = ThreadMetaRepository(session_factory)
|
||||
history_repo = TaskDeeplinkThreadRepository(session_factory)
|
||||
await thread_repo.create(
|
||||
"late-binding",
|
||||
user_id="creator",
|
||||
metadata={"taskId": "2055", "agent_id": "three-q-agent"},
|
||||
)
|
||||
assert await history_repo.list_threads("2055", "3qfx") == []
|
||||
|
||||
await thread_repo.update_metadata(
|
||||
"late-binding",
|
||||
{
|
||||
"taskRecordId": "2055",
|
||||
"taskGoPath": "3qfx",
|
||||
"thread_type": "task_deeplink",
|
||||
},
|
||||
user_id=None,
|
||||
)
|
||||
assert [row["thread_id"] for row in await history_repo.list_threads("2055", "3qfx")] == ["late-binding"]
|
||||
|
||||
await thread_repo.delete("late-binding", user_id=None)
|
||||
assert await history_repo.list_threads("2055", "3qfx") == []
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_mode_uses_the_same_dedicated_partition_contract():
|
||||
raw_store = InMemoryStore()
|
||||
thread_store = MemoryThreadMetaStore(raw_store)
|
||||
history_store = MemoryTaskDeeplinkThreadStore(raw_store, thread_store)
|
||||
|
||||
await thread_store.create(
|
||||
"memory-rwfx",
|
||||
user_id="creator-a",
|
||||
display_name="内存任务会话",
|
||||
metadata={
|
||||
"taskId": "108",
|
||||
"taskRecordId": "108",
|
||||
"taskGoPath": "rwfx",
|
||||
"thread_type": "task_deeplink",
|
||||
},
|
||||
)
|
||||
await thread_store.create(
|
||||
"memory-legacy",
|
||||
user_id="creator-b",
|
||||
metadata={"taskId": "108"},
|
||||
)
|
||||
await thread_store.create(
|
||||
"memory-xdfx",
|
||||
user_id="creator-c",
|
||||
metadata={
|
||||
"taskId": "108",
|
||||
"taskRecordId": "action-9",
|
||||
"taskGoPath": "xdfx",
|
||||
"thread_type": "task_deeplink",
|
||||
},
|
||||
)
|
||||
|
||||
rwfx = await history_store.list_threads("108", "rwfx")
|
||||
xdfx = await history_store.list_threads("action-9", "xdfx")
|
||||
assert [row["thread_id"] for row in rwfx] == ["memory-rwfx"]
|
||||
assert rwfx[0]["values"] == {"title": "内存任务会话"}
|
||||
assert [row["thread_id"] for row in xdfx] == ["memory-xdfx"]
|
||||
|
||||
await thread_store.delete("memory-rwfx", user_id=None)
|
||||
assert await history_store.list_threads("108", "rwfx") == []
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
"""Regression coverage for indexed task-workspace thread lookups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import sqlalchemy as sa
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_meta.sql import ThreadMetaRepository
|
||||
|
||||
|
||||
def _migration_config(database: Path) -> Config:
|
||||
migrations = Path(__file__).parents[3] / "packages" / "harness" / "deerflow" / "persistence" / "migrations"
|
||||
# Do not load alembic.ini's logging configuration in this focused unit
|
||||
# test: ``fileConfig`` disables pytest's capture handlers for later tests.
|
||||
config = Config()
|
||||
config.set_main_option("script_location", str(migrations))
|
||||
config.set_main_option("sqlalchemy.url", f"sqlite+aiosqlite:///{database.resolve().as_posix()}")
|
||||
return config
|
||||
|
||||
|
||||
def test_task_lookup_migration_backfills_existing_metadata_and_indexes(tmp_path: Path):
|
||||
"""A deployment already at the prior head gets usable task projections."""
|
||||
database = tmp_path / "thread-meta.db"
|
||||
engine = sa.create_engine(f"sqlite:///{database.resolve().as_posix()}")
|
||||
try:
|
||||
with engine.begin() as connection:
|
||||
connection.exec_driver_sql(
|
||||
"CREATE TABLE threads_meta ("
|
||||
"thread_id VARCHAR(64) PRIMARY KEY, assistant_id VARCHAR(128), user_id VARCHAR(64), "
|
||||
"display_name VARCHAR(256), status VARCHAR(20) NOT NULL, metadata_json JSON, "
|
||||
"created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL)"
|
||||
)
|
||||
connection.exec_driver_sql(
|
||||
"INSERT INTO threads_meta "
|
||||
"(thread_id, status, metadata_json, created_at, updated_at) VALUES "
|
||||
"('task-thread', 'idle', '{\"taskId\":\"108\",\"agent_id\":\"rwfx-agent\",\"actionId\":\"action-9\"}', "
|
||||
"CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"
|
||||
)
|
||||
connection.exec_driver_sql("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL PRIMARY KEY)")
|
||||
connection.exec_driver_sql("INSERT INTO alembic_version (version_num) VALUES ('20260920_02')")
|
||||
|
||||
command.upgrade(_migration_config(database), "head")
|
||||
|
||||
with engine.connect() as connection:
|
||||
columns = {column["name"] for column in sa.inspect(connection).get_columns("threads_meta")}
|
||||
indexes = {index["name"] for index in sa.inspect(connection).get_indexes("threads_meta")}
|
||||
assert {"task_id", "agent_id", "action_id"} <= columns
|
||||
assert {
|
||||
"ix_threads_meta_task_agent_updated",
|
||||
"ix_threads_meta_task_action_agent_updated",
|
||||
"ix_threads_meta_user_updated",
|
||||
} <= indexes
|
||||
assert connection.execute(
|
||||
sa.text("SELECT task_id, agent_id, action_id FROM threads_meta WHERE thread_id = 'task-thread'")
|
||||
).one() == ("108", "rwfx-agent", "action-9")
|
||||
# The new dedicated taskId + goPath history is intentionally not
|
||||
# backfilled from legacy task metadata.
|
||||
assert "task_deeplink_threads" in sa.inspect(connection).get_table_names()
|
||||
assert connection.execute(
|
||||
sa.text("SELECT COUNT(*) FROM task_deeplink_threads")
|
||||
).scalar_one() == 0
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_metadata_search_uses_indexed_projection_not_global_json_scan():
|
||||
"""A task with few chats must not page across unrelated thread metadata."""
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(ThreadMetaRow.__table__.create)
|
||||
|
||||
now = datetime.now(UTC)
|
||||
async with session_factory() as session:
|
||||
session.add_all(
|
||||
[
|
||||
ThreadMetaRow(
|
||||
thread_id=f"ordinary-{index:04d}",
|
||||
user_id="someone",
|
||||
metadata_json={"agent_id": "other-agent"},
|
||||
created_at=now + timedelta(seconds=index),
|
||||
updated_at=now + timedelta(seconds=index),
|
||||
)
|
||||
for index in range(600)
|
||||
]
|
||||
)
|
||||
session.add(
|
||||
ThreadMetaRow(
|
||||
thread_id="task-thread",
|
||||
user_id="another-user",
|
||||
task_id="108",
|
||||
agent_id="rwfx-agent",
|
||||
metadata_json={"taskId": "108", "agent_id": "rwfx-agent"},
|
||||
created_at=now - timedelta(days=1),
|
||||
updated_at=now - timedelta(days=1),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
statements: list[str] = []
|
||||
|
||||
def capture(_conn, _cursor, statement, _parameters, _context, _executemany):
|
||||
if "FROM threads_meta" in statement:
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(engine.sync_engine, "before_cursor_execute", capture)
|
||||
try:
|
||||
repo = ThreadMetaRepository(session_factory)
|
||||
rows = await repo.search(
|
||||
metadata={"taskId": "108", "agent_id": "rwfx-agent"},
|
||||
limit=101,
|
||||
user_id=None,
|
||||
exclude_system=True,
|
||||
)
|
||||
finally:
|
||||
event.remove(engine.sync_engine, "before_cursor_execute", capture)
|
||||
|
||||
assert [row["thread_id"] for row in rows] == ["task-thread"]
|
||||
# The old implementation needs two global 500-row batches for this
|
||||
# fixture. The projected columns make one task-scoped SELECT enough.
|
||||
assert len(statements) == 1
|
||||
assert "threads_meta.task_id" in statements[0]
|
||||
assert "threads_meta.agent_id" in statements[0]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_metadata_projections_are_created_updated_and_counted():
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(ThreadMetaRow.__table__.create)
|
||||
repo = ThreadMetaRepository(session_factory)
|
||||
|
||||
await repo.create(
|
||||
"task-thread",
|
||||
user_id=None,
|
||||
metadata={"taskId": "108", "agent_id": "rwfx-agent"},
|
||||
)
|
||||
await repo.update_metadata("task-thread", {"actionId": "action-9"}, user_id=None)
|
||||
|
||||
async with session_factory() as session:
|
||||
row = await session.get(ThreadMetaRow, "task-thread")
|
||||
assert row is not None
|
||||
assert (row.task_id, row.agent_id, row.action_id) == ("108", "rwfx-agent", "action-9")
|
||||
|
||||
metadata = {"taskId": "108", "agent_id": "rwfx-agent", "actionId": "action-9"}
|
||||
rows = await repo.search(metadata=metadata, limit=10, user_id=None, exclude_system=True)
|
||||
total = await repo.count(metadata=metadata, user_id=None, exclude_system=True)
|
||||
|
||||
assert [row["thread_id"] for row in rows] == ["task-thread"]
|
||||
assert total == 1
|
||||
finally:
|
||||
await engine.dispose()
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
"""No-model concurrency diagnostics using actual HTTP routes and SQLite saver."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
|
||||
from app.gateway.operation_timing import OperationTiming
|
||||
from app.gateway.routers.threads import ThreadCreateRequest, ThreadSearchRequest, create_thread, router, search_threads
|
||||
|
||||
|
||||
class ThreadStore:
|
||||
def __init__(self):
|
||||
self.rows = {}
|
||||
|
||||
async def get(self, thread_id):
|
||||
return self.rows.get(thread_id)
|
||||
|
||||
async def create(self, thread_id, **kwargs):
|
||||
self.rows[thread_id] = {"thread_id": thread_id, "metadata": kwargs.get("metadata", {})}
|
||||
return self.rows[thread_id]
|
||||
|
||||
async def search(self, **kwargs):
|
||||
return list(self.rows.values())
|
||||
|
||||
|
||||
def client_for(saver, store=None):
|
||||
app = FastAPI()
|
||||
app.state.checkpointer = saver
|
||||
app.state.thread_store = store or ThreadStore()
|
||||
app.include_router(router)
|
||||
return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test")
|
||||
|
||||
|
||||
def phase_ms(response):
|
||||
return {name: float(value) for name, value in (item.strip().split(";dur=") for item in response.headers["server-timing"].split(","))}
|
||||
|
||||
|
||||
def test_twenty_creates_preserve_immediate_checkpoints_and_do_not_log_content(caplog):
|
||||
async def run():
|
||||
saver = InMemorySaver()
|
||||
async with client_for(saver) as client:
|
||||
responses = await asyncio.gather(*(client.post("/api/threads", json={"metadata": {"private_content": "do-not-log-me"}}) for _ in range(20)))
|
||||
ids = set()
|
||||
for response in responses:
|
||||
assert response.status_code == 200
|
||||
thread_id = response.json()["thread_id"]
|
||||
ids.add(thread_id)
|
||||
assert set(phase_ms(response)) == {"metadata_lookup", "metadata_create", "checkpoint_create", "handler_total"}
|
||||
checkpoint = await saver.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
assert checkpoint is not None
|
||||
assert checkpoint.checkpoint["channel_values"] == {}
|
||||
assert len(ids) == 20
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
asyncio.run(run())
|
||||
assert "GATEWAY_OPERATION_TIMING operation=thread_create outcome=ok" in caplog.text
|
||||
assert "checkpointer=InMemorySaver" in caplog.text
|
||||
assert "do-not-log-me" not in caplog.text
|
||||
|
||||
|
||||
def test_direct_internal_callers_do_not_need_a_response_argument():
|
||||
async def run():
|
||||
app = FastAPI()
|
||||
app.state.checkpointer = InMemorySaver()
|
||||
app.state.thread_store = ThreadStore()
|
||||
request = Request({"type": "http", "app": app})
|
||||
created = await create_thread(body=ThreadCreateRequest(), request=request)
|
||||
found = await search_threads(body=ThreadSearchRequest(), request=request)
|
||||
assert [item.thread_id for item in found] == [created.thread_id]
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_analysis_project_calls_real_create_thread_without_response(monkeypatch):
|
||||
from app.gateway.routers import analysis_projects
|
||||
|
||||
async def run():
|
||||
project = {"id": "project", "user_id": "owner", "mode": "qa", "name": "Test", "agent_id": None}
|
||||
project_store = SimpleNamespace(bind_thread=AsyncMock())
|
||||
monkeypatch.setattr(analysis_projects, "owned_project", AsyncMock(return_value=project))
|
||||
monkeypatch.setattr(analysis_projects, "project_store", lambda request: project_store)
|
||||
monkeypatch.setattr(analysis_projects, "validate_project_agent", AsyncMock())
|
||||
app = FastAPI()
|
||||
app.state.checkpointer = InMemorySaver()
|
||||
app.state.thread_store = ThreadStore()
|
||||
request = Request({"type": "http", "app": app})
|
||||
created = await analysis_projects.new_project_thread("project", request)
|
||||
assert created.metadata["analysis_project_id"] == "project"
|
||||
assert created.thread_id in app.state.thread_store.rows
|
||||
project_store.bind_thread.assert_awaited_once()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["capture", "ask"])
|
||||
def test_browser_context_calls_real_create_thread_without_response(monkeypatch, tmp_path, kind):
|
||||
from app.gateway.routers import browser_context
|
||||
from deerflow.config.browser_context_config import BrowserContextConfig
|
||||
|
||||
async def run():
|
||||
app = FastAPI()
|
||||
app.state.checkpointer = InMemorySaver()
|
||||
app.state.thread_store = ThreadStore()
|
||||
request = Request({"type": "http", "app": app})
|
||||
config = SimpleNamespace(browser_context=BrowserContextConfig(enabled=True))
|
||||
if kind == "capture":
|
||||
monkeypatch.setattr(browser_context, "ensure_uploads_dir", lambda thread_id: tmp_path)
|
||||
monkeypatch.setattr(browser_context, "_sync_to_sandbox_if_needed", AsyncMock())
|
||||
body = browser_context.BrowserContextCaptureRequest(content="Page body", title="Test")
|
||||
result = await browser_context.capture_browser_context.__wrapped__(body, request, config)
|
||||
else:
|
||||
monkeypatch.setattr(browser_context, "start_run", AsyncMock(return_value=SimpleNamespace(task=None, run_id="test-run", error=None)))
|
||||
body = browser_context.BrowserContextAskRequest(content="Page body", question="Summarize")
|
||||
result = await browser_context.ask_browser_context.__wrapped__(body, request, config)
|
||||
assert result.success
|
||||
assert result.thread_id in app.state.thread_store.rows
|
||||
assert await app.state.checkpointer.aget_tuple({"configurable": {"thread_id": result.thread_id}}) is not None
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_timing_handler_failure_does_not_replace_request_error():
|
||||
log = Mock()
|
||||
log.warning.side_effect = RuntimeError("broken log destination")
|
||||
failure = HTTPException(409, "original error", headers={"X-Test": "retained"})
|
||||
with pytest.raises(HTTPException) as captured:
|
||||
with OperationTiming(None, log, "test") as timing, timing.phase("metadata_create"):
|
||||
raise failure
|
||||
assert captured.value is failure
|
||||
assert failure.headers["X-Test"] == "retained"
|
||||
assert "Server-Timing" in failure.headers
|
||||
|
||||
|
||||
def test_sqlite_history_lock_blocks_create_but_not_metadata_search_or_idempotency(caplog):
|
||||
async def run():
|
||||
async with AsyncSqliteSaver.from_conn_string(":memory:") as saver:
|
||||
await saver.aput({"configurable": {"thread_id": "existing", "checkpoint_ns": ""}}, empty_checkpoint(), {}, {})
|
||||
store = ThreadStore()
|
||||
await store.create("existing")
|
||||
# Installed SQLite saver holds its lock while alist is suspended
|
||||
# at yield, as can happen during checkpoint-history consumption.
|
||||
history = saver.alist({"configurable": {"thread_id": "existing"}})
|
||||
await anext(history)
|
||||
assert saver.lock.locked()
|
||||
async with client_for(saver, store) as client:
|
||||
create = asyncio.create_task(client.post("/api/threads", json={"thread_id": "new"}))
|
||||
try:
|
||||
search = await asyncio.wait_for(client.post("/api/threads/search", json={}), timeout=2)
|
||||
existing = await asyncio.wait_for(client.post("/api/threads", json={"thread_id": "existing"}), timeout=2)
|
||||
assert search.status_code == existing.status_code == 200
|
||||
assert set(phase_ms(search)) == {"metadata_search", "handler_total"}
|
||||
assert set(phase_ms(existing)) == {"metadata_lookup", "handler_total"}
|
||||
await asyncio.sleep(0.06)
|
||||
assert not create.done()
|
||||
assert "new" in store.rows # Metadata creation already finished.
|
||||
finally:
|
||||
await history.aclose()
|
||||
response = await asyncio.wait_for(create, timeout=2)
|
||||
phases = phase_ms(response)
|
||||
assert response.status_code == 200
|
||||
assert phases["checkpoint_create"] >= 40
|
||||
assert phases["checkpoint_create"] > phases["metadata_create"]
|
||||
assert await saver.aget_tuple({"configurable": {"thread_id": "new"}}) is not None
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
asyncio.run(run())
|
||||
assert "checkpointer=AsyncSqliteSaver" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["metadata_create", "checkpoint_create"])
|
||||
def test_failures_keep_timing_headers_and_existing_error_contract(stage, caplog):
|
||||
class FailingStore(ThreadStore):
|
||||
async def create(self, thread_id, **kwargs):
|
||||
raise RuntimeError("synthetic metadata failure")
|
||||
|
||||
class FailingSaver:
|
||||
async def aput(self, *args):
|
||||
raise RuntimeError("synthetic checkpoint failure")
|
||||
|
||||
async def run():
|
||||
saver = InMemorySaver() if stage == "metadata_create" else FailingSaver()
|
||||
store = FailingStore() if stage == "metadata_create" else ThreadStore()
|
||||
async with client_for(saver, store) as client:
|
||||
response = await client.post("/api/threads", json={})
|
||||
assert response.status_code == 500
|
||||
assert response.json() == {"detail": "Failed to create thread"}
|
||||
assert stage in phase_ms(response)
|
||||
|
||||
asyncio.run(run())
|
||||
assert "outcome=error" in caplog.text
|
||||
|
||||
|
||||
def test_installed_postgres_saver_serializes_cursor_checkout_even_with_pool(monkeypatch):
|
||||
"""Characterize the installed saver, without contacting PostgreSQL."""
|
||||
from langgraph.checkpoint.postgres import aio
|
||||
|
||||
async def run():
|
||||
pool = object()
|
||||
entered = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
checkouts = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def cursor(**kwargs):
|
||||
yield object()
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_connection(conn):
|
||||
nonlocal checkouts
|
||||
assert conn is pool
|
||||
checkouts += 1
|
||||
yield SimpleNamespace(cursor=cursor)
|
||||
|
||||
monkeypatch.setattr(aio._ainternal, "get_connection", get_connection)
|
||||
saver = aio.AsyncPostgresSaver(pool)
|
||||
|
||||
async def first():
|
||||
async with saver._cursor():
|
||||
entered.set()
|
||||
await release.wait()
|
||||
|
||||
async def second():
|
||||
async with saver._cursor():
|
||||
pass
|
||||
|
||||
first_task = asyncio.create_task(first())
|
||||
await entered.wait()
|
||||
second_task = asyncio.create_task(second())
|
||||
await asyncio.sleep(0.02)
|
||||
assert checkouts == 1
|
||||
assert not second_task.done()
|
||||
release.set()
|
||||
await asyncio.gather(first_task, second_task)
|
||||
assert checkouts == 2
|
||||
|
||||
asyncio.run(run())
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Todo control messages stay available to the model without becoming user turns."""
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
from deerflow.agents.middlewares.todo_middleware import TodoMiddleware
|
||||
from deerflow.runtime.serialization import serialize_channel_values, serialize_messages_tuple
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["before", "after"])
|
||||
def test_todo_reminders_keep_control_semantics_and_hide_after_serialization(phase):
|
||||
middleware = TodoMiddleware()
|
||||
state = {
|
||||
"messages": [HumanMessage(content="Draw a map"), AIMessage(content="The map is ready")],
|
||||
"todos": [{"content": "Finish the map", "status": "in_progress"}],
|
||||
}
|
||||
result = getattr(middleware, f"{phase}_model")(state, None)
|
||||
assert result is not None
|
||||
reminder = result["messages"][0]
|
||||
assert isinstance(reminder, HumanMessage)
|
||||
assert "Finish the map" in reminder.content
|
||||
assert reminder.name == ("todo_reminder" if phase == "before" else "todo_completion_reminder")
|
||||
if phase == "after":
|
||||
assert result["jump_to"] == "model"
|
||||
|
||||
for serialized in (
|
||||
serialize_channel_values({"messages": [reminder]})["messages"][0],
|
||||
serialize_messages_tuple((reminder, {}))[0],
|
||||
):
|
||||
assert serialized["additional_kwargs"]["hide_from_ui"] is True
|
||||
assert serialized["additional_kwargs"]["lc_source"] == "todo"
|
||||
|
||||
# Hiding a reminder must not break the existing deduplication/retry bound.
|
||||
if phase == "before":
|
||||
state["messages"].append(reminder)
|
||||
else:
|
||||
state["messages"].extend([reminder, reminder])
|
||||
assert getattr(middleware, f"{phase}_model")(state, None) is None
|
||||
|
||||
|
||||
def test_completed_todos_do_not_create_a_completion_reminder():
|
||||
assert TodoMiddleware().after_model(
|
||||
{"messages": [AIMessage(content="Done")], "todos": [{"content": "Draw a map", "status": "completed"}]},
|
||||
None,
|
||||
) is None
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from deerflow.agents.middlewares.tool_metrics_middleware import ToolMetricsMiddleware
|
||||
|
||||
|
||||
def _request(name: str):
|
||||
return SimpleNamespace(tool_call={"id": f"call-{name}", "name": name, "args": {}})
|
||||
|
||||
|
||||
def _result(name: str) -> ToolMessage:
|
||||
return ToolMessage(content="OK", name=name, tool_call_id=f"call-{name}")
|
||||
|
||||
|
||||
def test_internal_tool_call_is_excluded_from_metrics(monkeypatch) -> None:
|
||||
middleware = ToolMetricsMiddleware()
|
||||
flush = Mock()
|
||||
monkeypatch.setattr(middleware, "_flush", flush)
|
||||
handler = Mock(return_value=_result("begin_write_file"))
|
||||
|
||||
result = middleware.wrap_tool_call(_request("begin_write_file"), handler)
|
||||
|
||||
assert result == handler.return_value
|
||||
handler.assert_called_once()
|
||||
flush.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_async_tool_call_is_excluded_from_metrics(monkeypatch) -> None:
|
||||
middleware = ToolMetricsMiddleware()
|
||||
flush = Mock()
|
||||
monkeypatch.setattr(middleware, "_flush", flush)
|
||||
handler = AsyncMock(return_value=_result("begin_write_file"))
|
||||
|
||||
result = await middleware.awrap_tool_call(_request("begin_write_file"), handler)
|
||||
|
||||
assert result == handler.return_value
|
||||
handler.assert_awaited_once()
|
||||
flush.assert_not_called()
|
||||
|
||||
|
||||
def test_canonical_write_file_is_still_recorded(monkeypatch) -> None:
|
||||
middleware = ToolMetricsMiddleware()
|
||||
flush = Mock()
|
||||
monkeypatch.setattr(middleware, "_flush", flush)
|
||||
|
||||
middleware.wrap_tool_call(_request("write_file"), lambda _: _result("write_file"))
|
||||
|
||||
flush.assert_called_once()
|
||||
Reference in new issue
Block a user