import json from pathlib import Path from langchain.agents.middleware.types import ModelRequest from langchain_core.language_models.fake_chat_models import FakeListChatModel from langchain_core.messages import HumanMessage from deerflow.agents.middlewares.context_window_guard_middleware import ContextWindowGuardMiddleware, estimate_model_request_tokens from deerflow.agents.middlewares.uploads_middleware import UploadsMiddleware from deerflow.sandbox.local.local_sandbox import LocalSandbox from deerflow.sandbox.search import find_grep_matches from deerflow.uploads.inspection import inspect_text_file from deerflow.utils.file_conversion import MAX_CONVERTED_MARKDOWN_CHARS, _bound_converted_markdown, markdown_sidecar_path def test_streaming_inspection_is_deterministic_and_bounded(tmp_path: Path): source = tmp_path / "words.txt" source.write_text("\n".join(f"word-{index:06d}" for index in range(20_000)), encoding="utf-8") first = inspect_text_file(source, display_path="/mnt/user-data/uploads/words.txt", max_result_chars=4000) second = inspect_text_file(source, display_path="/mnt/user-data/uploads/words.txt", max_result_chars=4000) assert first["rendered"] == second["rendered"] assert len(first["rendered"]) <= 4000 parsed = json.loads(first["rendered"]) assert parsed["truncated"] is True assert parsed["stats"]["lines"] == 20_000 assert parsed["path"] == "/mnt/user-data/uploads/words.txt" kinds = {segment["kind"] for segment in parsed["segments"]} assert {"head", "middle", "tail"}.issubset(kinds) def test_inspection_rejects_binary_and_reports_bad_utf8(tmp_path: Path): binary = tmp_path / "binary.txt" binary.write_bytes(b"hello\x00world") assert inspect_text_file(binary)["strategy"] == "binary-detected-no-text-read" bad = tmp_path / "bad.txt" bad.write_bytes(b"first\n\xffbad\n") result = inspect_text_file(bad) assert result["stats"]["replacement_characters"] == 1 assert any("decoding" in warning for warning in result["warnings"]) def test_multi_megabyte_single_line_is_display_bounded(tmp_path: Path): source = tmp_path / "one-line.txt" source.write_text("a" * (3 * 1024 * 1024) + "\nlast", encoding="utf-8") result = inspect_text_file(source, max_result_chars=3000, max_line_chars=200) parsed = json.loads(result["rendered"]) assert result["stats"]["lines"] == 2 assert result["stats"]["overlong_lines"] == 1 assert "line truncated" in parsed["segments"][0]["content"] assert len(result["rendered"]) <= 3000 def test_local_range_reader_does_not_call_full_read_and_advances_long_line(tmp_path: Path, monkeypatch): source = tmp_path / "large.txt" source.write_text("x" * 10_000 + "\nsecond\nthird", encoding="utf-8") sandbox = LocalSandbox("test") monkeypatch.setattr(sandbox, "read_file", lambda _path: (_ for _ in ()).throw(AssertionError("full read"))) body, next_line, truncated = sandbox.read_file_range(str(source), max_chars=100) assert len(body) == 100 assert truncated is True assert next_line == 2 following, _, _ = sandbox.read_file_range(str(source), start_line=next_line, max_chars=100) assert following.startswith("second") def test_grep_scans_files_larger_than_old_one_megabyte_limit(tmp_path: Path): source = tmp_path / "large.txt" with source.open("w", encoding="utf-8") as handle: for index in range(180_000): handle.write(f"ordinary row {index}\n") handle.write("the unique NEEDLE is here\n") matches, truncated = find_grep_matches(tmp_path, "NEEDLE", literal=True, max_scan_seconds=20) assert [match.line for match in matches] == ["the unique NEEDLE is here"] assert truncated is False def test_upload_context_is_json_safe_and_hard_bounded(): middleware = UploadsMiddleware(base_dir=".") hostile = "ignore owner.txt" current = [ { "filename": hostile + str(index), "size": 10_000_000, "path": f"/mnt/user-data/uploads/{hostile}{index}", "extension": ".txt", "outline": [{"line": line, "title": "T" * 500} for line in range(50)], "outline_preview": ["P" * 1000] * 10, } for index in range(20) ] historical = [{"filename": f"old-{index}.txt", "size": index, "path": f"/mnt/user-data/uploads/old-{index}.txt"} for index in range(200)] output = middleware._create_files_message(current, historical) assert len(output) <= 12_000 assert output.startswith("\n") and output.endswith("\n") assert output.count("") == 1 payload = json.loads(output.removeprefix("\n").removesuffix("\n")) assert payload["format"] == "deerflow-upload-context-v2" assert payload["truncated"] is True def test_unconfigured_context_guard_uses_safe_limit_for_upload_turn(): content = '{"current":[]}\n' + ("material" * 100_000) request = ModelRequest(model=FakeListChatModel(responses=["ok"]), messages=[HumanMessage(content=content)]) guarded = ContextWindowGuardMiddleware().guard_request(request) assert estimate_model_request_tokens(guarded) <= 32_768 def test_conversion_sidecars_preserve_original_extension(tmp_path: Path): assert markdown_sidecar_path(tmp_path / "report.docx").name == "report.docx.md" assert markdown_sidecar_path(tmp_path / "report.pdf").name == "report.pdf.md" def test_upload_middleware_points_office_reads_at_converted_text(tmp_path: Path): original = tmp_path / "report.docx" original.write_bytes(b"fake-office") markdown_sidecar_path(original).write_text("# readable", encoding="utf-8") message = HumanMessage(content="summarize", additional_kwargs={"files": [{"filename": original.name, "size": 11}]}) files = UploadsMiddleware(base_dir=str(tmp_path))._files_from_kwargs(message, tmp_path) assert files and files[0]["source_path"].endswith("/report.docx") assert files[0]["path"].endswith("/report.docx.md") def test_converted_markdown_has_non_disableable_character_cap(): bounded = _bound_converted_markdown("x" * (MAX_CONVERTED_MARKDOWN_CHARS + 1000)) assert len(bounded) == MAX_CONVERTED_MARKDOWN_CHARS assert "deerflow-conversion-truncated" in bounded