sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
+57
-12
@@ -21,6 +21,7 @@ import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
@@ -55,17 +56,46 @@ _LIBREOFFICE_TARGETS = {
|
||||
}
|
||||
_LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
|
||||
# Files larger than this threshold are converted in a background thread.
|
||||
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
|
||||
# scheduling overhead for them.
|
||||
_ASYNC_THRESHOLD_BYTES = 1 * 1024 * 1024 # 1 MB
|
||||
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
|
||||
_CONVERSION_CONCURRENCY = 2
|
||||
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
|
||||
|
||||
# If pymupdf4llm produces fewer characters *per page* than this threshold,
|
||||
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
|
||||
# Rationale: normal text PDFs yield 200-2000 chars/page; image-based PDFs
|
||||
# yield close to 0. 50 chars/page gives a wide safety margin.
|
||||
# Falls back to absolute 200-char check when page count is unavailable.
|
||||
_MIN_CHARS_PER_PAGE = 50
|
||||
_MIN_CHARS_PER_PAGE = 50
|
||||
|
||||
|
||||
def markdown_sidecar_path(file_path: Path) -> Path:
|
||||
"""Return the collision-free Markdown companion path for a document.
|
||||
|
||||
Keeping the original extension (``report.docx.md``) avoids the historical
|
||||
collision where ``report.docx`` and ``report.pdf`` both overwrote
|
||||
``report.md``.
|
||||
"""
|
||||
|
||||
return file_path.with_name(f"{file_path.name}.md")
|
||||
|
||||
|
||||
def find_markdown_sidecar(file_path: Path) -> Path | None:
|
||||
"""Find a new-style sidecar, falling back to the legacy ``<stem>.md``."""
|
||||
|
||||
# Only formats handled by the conversion pipeline can own a generated
|
||||
# Markdown companion. Without this guard, unrelated uploads such as
|
||||
# ``report.txt`` or ``report.jpg`` incorrectly claimed a real user upload
|
||||
# named ``report.md`` as their legacy sidecar.
|
||||
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
|
||||
return None
|
||||
|
||||
current = markdown_sidecar_path(file_path)
|
||||
if current.is_file():
|
||||
return current
|
||||
legacy = file_path.with_suffix(".md")
|
||||
if legacy != file_path and legacy.is_file():
|
||||
return legacy
|
||||
return None
|
||||
|
||||
|
||||
def _pymupdf_output_too_sparse(text: str, file_path: Path) -> bool:
|
||||
@@ -187,6 +217,17 @@ def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
|
||||
return _convert_with_markitdown(converted_path)
|
||||
|
||||
|
||||
def _bound_converted_markdown(text: str) -> str:
|
||||
"""Apply a non-disableable cap before converted text is persisted or returned."""
|
||||
if len(text) <= MAX_CONVERTED_MARKDOWN_CHARS:
|
||||
return text
|
||||
marker = (
|
||||
"\n\n<!-- deerflow-conversion-truncated: extracted text exceeded "
|
||||
f"{MAX_CONVERTED_MARKDOWN_CHARS} characters; use the original file for targeted inspection -->\n"
|
||||
)
|
||||
return text[: MAX_CONVERTED_MARKDOWN_CHARS - len(marker)] + marker
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
|
||||
@@ -194,6 +235,12 @@ def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
with _CONVERSION_SEMAPHORE:
|
||||
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
|
||||
|
||||
|
||||
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Parser implementation; callers must use :func:`_do_convert`."""
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
|
||||
@@ -239,14 +286,12 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
"""
|
||||
try:
|
||||
pdf_converter = _get_pdf_converter()
|
||||
file_size = file_path.stat().st_size
|
||||
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
|
||||
# large factor even for a small upload. Always keep them off the event
|
||||
# loop; the synchronous worker has a bounded global concurrency gate.
|
||||
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
|
||||
|
||||
if file_size > _ASYNC_THRESHOLD_BYTES:
|
||||
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
|
||||
else:
|
||||
text = _do_convert(file_path, pdf_converter)
|
||||
|
||||
md_path = file_path.with_suffix(".md")
|
||||
md_path = markdown_sidecar_path(file_path)
|
||||
md_path.write_text(text, encoding="utf-8")
|
||||
|
||||
logger.info("Converted %s to markdown: %s (%d chars)", file_path.name, md_path.name, len(text))
|
||||
|
||||
@@ -16,14 +16,34 @@ class InlineThinkTagFilter:
|
||||
not safe: it can leak a partial tag or the model's reasoning into the
|
||||
visible draft. This small state machine holds only the suffix that might
|
||||
still form a tag and emits all confirmed answer text immediately.
|
||||
|
||||
``leading_only=True`` reserves only one optional response-leading envelope
|
||||
for reasoning. It is suitable for exact payloads such as file bodies,
|
||||
where a later literal ``<think>`` must remain ordinary content.
|
||||
"""
|
||||
|
||||
_OPEN_TAG = "<think>"
|
||||
_CLOSE_TAG = "</think>"
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
allow_implicit_close: bool = False,
|
||||
leading_only: bool = False,
|
||||
) -> None:
|
||||
self._inside_think = False
|
||||
self._pending = ""
|
||||
self._channel_decided = not allow_implicit_close
|
||||
# File-body transports need a stricter interpretation than ordinary
|
||||
# chat rendering: only one optional reasoning envelope at the start of
|
||||
# the response is protocol metadata. Once file bytes begin, a literal
|
||||
# ``<think>`` is user content and must never be rewritten.
|
||||
self._leading_only = leading_only
|
||||
self._leading_phase = (
|
||||
"implicit_probe"
|
||||
if leading_only and allow_implicit_close
|
||||
else ("probe" if leading_only else "disabled")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _possible_tag_prefix_size(content: str, tag: str) -> int:
|
||||
@@ -39,11 +59,31 @@ class InlineThinkTagFilter:
|
||||
if not chunk:
|
||||
return "", ""
|
||||
|
||||
if self._leading_only:
|
||||
return self._push_leading_parts(chunk)
|
||||
|
||||
remaining = self._pending + chunk
|
||||
self._pending = ""
|
||||
visible: list[str] = []
|
||||
thinking: list[str] = []
|
||||
|
||||
if not self._channel_decided:
|
||||
lowered = remaining.lower()
|
||||
open_at = lowered.find(self._OPEN_TAG)
|
||||
close_at = lowered.find(self._CLOSE_TAG)
|
||||
if close_at >= 0 and (open_at < 0 or close_at < open_at):
|
||||
# Some templates inject the opening tag outside returned
|
||||
# content. Keep the ambiguous prefix private until its close
|
||||
# marker arrives, even when that marker spans chunks.
|
||||
thinking.append(remaining[:close_at])
|
||||
remaining = remaining[close_at + len(self._CLOSE_TAG) :]
|
||||
self._channel_decided = True
|
||||
elif open_at >= 0:
|
||||
self._channel_decided = True
|
||||
else:
|
||||
self._pending = remaining
|
||||
return "", ""
|
||||
|
||||
while remaining:
|
||||
tag = self._CLOSE_TAG if self._inside_think else self._OPEN_TAG
|
||||
marker_at = remaining.lower().find(tag)
|
||||
@@ -61,12 +101,116 @@ class InlineThinkTagFilter:
|
||||
|
||||
return "".join(visible), "".join(thinking)
|
||||
|
||||
def _push_leading_parts(self, chunk: str) -> tuple[str, str]:
|
||||
"""Route only a leading ``<think>`` envelope as provider reasoning.
|
||||
|
||||
This mode is intended for protocol payloads such as streamed file
|
||||
bodies. It waits only while the response prefix could still become an
|
||||
opening tag, supports tags split across arbitrary chunks, and locks to
|
||||
the visible channel immediately after the closing tag or the first
|
||||
confirmed file byte. Later literal tags are therefore preserved.
|
||||
"""
|
||||
|
||||
remaining = self._pending + chunk
|
||||
self._pending = ""
|
||||
|
||||
if self._leading_phase == "visible":
|
||||
return remaining, ""
|
||||
|
||||
if self._leading_phase in {"probe", "implicit_probe"}:
|
||||
candidate = remaining.lstrip()
|
||||
lowered = candidate.lower()
|
||||
if lowered.startswith(self._OPEN_TAG):
|
||||
# Whitespace before a leading reasoning envelope belongs to
|
||||
# the provider's presentation, not to the prepared file.
|
||||
remaining = candidate[len(self._OPEN_TAG) :]
|
||||
self._leading_phase = "thinking"
|
||||
self._inside_think = True
|
||||
elif self._OPEN_TAG.startswith(lowered):
|
||||
# Empty/whitespace-only prefixes and partial tags remain
|
||||
# ambiguous until the next provider chunk arrives.
|
||||
self._pending = remaining
|
||||
return "", ""
|
||||
elif self._leading_phase == "implicit_probe":
|
||||
close_at = remaining.lower().find(self._CLOSE_TAG)
|
||||
if close_at >= 0:
|
||||
# Some chat templates inject ``<think>`` outside the
|
||||
# returned token stream. This mode is enabled only when
|
||||
# the immediately preceding bridge-selection response
|
||||
# demonstrated that exact close-only convention. Until
|
||||
# the closing marker arrives the prefix is intentionally
|
||||
# buffered: ordinary file text and implicit reasoning are
|
||||
# otherwise indistinguishable.
|
||||
thinking = remaining[:close_at]
|
||||
visible = remaining[close_at + len(self._CLOSE_TAG) :]
|
||||
self._leading_phase = "visible"
|
||||
return visible, thinking
|
||||
self._pending = remaining
|
||||
return "", ""
|
||||
else:
|
||||
self._leading_phase = "visible"
|
||||
return remaining, ""
|
||||
|
||||
marker_at = remaining.lower().find(self._CLOSE_TAG)
|
||||
if marker_at >= 0:
|
||||
thinking = remaining[:marker_at]
|
||||
visible = remaining[marker_at + len(self._CLOSE_TAG) :]
|
||||
self._leading_phase = "visible"
|
||||
self._inside_think = False
|
||||
return visible, thinking
|
||||
|
||||
suffix_size = self._possible_tag_prefix_size(remaining, self._CLOSE_TAG)
|
||||
thinking = remaining[:-suffix_size] if suffix_size else remaining
|
||||
self._pending = remaining[-suffix_size:] if suffix_size else ""
|
||||
return "", thinking
|
||||
|
||||
def push(self, chunk: str) -> str:
|
||||
"""Compatibility helper returning only confirmed visible text."""
|
||||
return self.push_parts(chunk)[0]
|
||||
|
||||
def confirm_visible_channel(self) -> str:
|
||||
"""Stop close-only probing and return any buffered visible prelude.
|
||||
|
||||
A provider that emits a dedicated reasoning field has already told us
|
||||
how to classify its trace. Its ordinary ``content`` channel can then
|
||||
stream immediately even when the model supports thinking.
|
||||
"""
|
||||
if self._leading_only:
|
||||
if self._leading_phase != "probe":
|
||||
if self._leading_phase != "implicit_probe":
|
||||
return ""
|
||||
self._leading_phase = "visible"
|
||||
pending = self._pending
|
||||
self._pending = ""
|
||||
return pending
|
||||
if self._channel_decided:
|
||||
return ""
|
||||
self._channel_decided = True
|
||||
pending = self._pending
|
||||
self._pending = ""
|
||||
return pending
|
||||
|
||||
def finish_parts(self) -> tuple[str, str]:
|
||||
"""Flush the remaining visible or thinking text at end of stream."""
|
||||
if self._leading_only:
|
||||
tail = self._pending
|
||||
self._pending = ""
|
||||
if self._leading_phase == "implicit_probe":
|
||||
# No close-only marker ever arrived. The buffered bytes are a
|
||||
# normal file body, not reasoning; preserve them exactly.
|
||||
self._leading_phase = "visible"
|
||||
return tail, ""
|
||||
if self._leading_phase == "thinking":
|
||||
self._inside_think = False
|
||||
self._leading_phase = "visible"
|
||||
return "", tail
|
||||
self._leading_phase = "visible"
|
||||
return tail, ""
|
||||
if not self._channel_decided:
|
||||
tail = self._pending
|
||||
self._pending = ""
|
||||
self._channel_decided = True
|
||||
return tail, ""
|
||||
if self._inside_think:
|
||||
thinking = self._pending
|
||||
self._pending = ""
|
||||
@@ -80,6 +224,14 @@ class InlineThinkTagFilter:
|
||||
"""Compatibility helper returning only the visible stream tail."""
|
||||
return self.finish_parts()[0]
|
||||
|
||||
@property
|
||||
def has_unclosed_thinking(self) -> bool:
|
||||
"""Whether the stream is currently inside a reasoning envelope."""
|
||||
|
||||
if self._leading_only:
|
||||
return self._leading_phase == "thinking"
|
||||
return self._inside_think
|
||||
|
||||
|
||||
def strip_inline_think_tags(content: str) -> str:
|
||||
"""Return complete text with inline model-thinking blocks removed."""
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Visibility rules for transport-only tools used inside the agent graph.
|
||||
|
||||
The graph/checkpoint must retain these calls because they are part of the
|
||||
model's causal history. Public serializers use the projection helpers below
|
||||
without mutating that history.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
INTERNAL_TOOL_NAMES = frozenset({"begin_write_file"})
|
||||
|
||||
|
||||
def is_internal_tool_name(name: object) -> bool:
|
||||
"""Return whether *name* is an exact match for a transport-only tool."""
|
||||
|
||||
return isinstance(name, str) and name in INTERNAL_TOOL_NAMES
|
||||
|
||||
|
||||
def _serialized_tool_name(tool_call: object) -> object:
|
||||
if not isinstance(tool_call, Mapping):
|
||||
return None
|
||||
name = tool_call.get("name")
|
||||
if name is not None:
|
||||
return name
|
||||
function = tool_call.get("function")
|
||||
return function.get("name") if isinstance(function, Mapping) else None
|
||||
|
||||
|
||||
def internal_tool_call_ids(messages: Iterable[object]) -> set[str]:
|
||||
"""Collect ids belonging to exact internal calls in serialized messages."""
|
||||
|
||||
ids: set[str] = set()
|
||||
for message in messages:
|
||||
if not isinstance(message, Mapping):
|
||||
continue
|
||||
tool_calls = message.get("tool_calls")
|
||||
if not isinstance(tool_calls, (list, tuple)):
|
||||
continue
|
||||
for tool_call in tool_calls:
|
||||
if not is_internal_tool_name(_serialized_tool_name(tool_call)):
|
||||
continue
|
||||
call_id = tool_call.get("id") if isinstance(tool_call, Mapping) else None
|
||||
if call_id:
|
||||
ids.add(str(call_id))
|
||||
return ids
|
||||
|
||||
|
||||
def project_outward_messages(
|
||||
messages: Iterable[Any],
|
||||
*,
|
||||
known_internal_tool_call_ids: Iterable[str] = (),
|
||||
) -> list[Any]:
|
||||
"""Remove transport-only call/result pairs from serialized messages."""
|
||||
|
||||
materialized = list(messages)
|
||||
internal_ids = {str(call_id) for call_id in known_internal_tool_call_ids if call_id}
|
||||
projected: list[Any] = []
|
||||
|
||||
for message in materialized:
|
||||
if not isinstance(message, Mapping):
|
||||
projected.append(message)
|
||||
continue
|
||||
|
||||
kind = str(message.get("type") or message.get("role") or "").lower()
|
||||
if kind in {"ai", "assistant", "aimessage", "aimessagechunk"}:
|
||||
tool_calls = message.get("tool_calls")
|
||||
if isinstance(tool_calls, (list, tuple)) and tool_calls:
|
||||
internal_ids.update(internal_tool_call_ids([message]))
|
||||
visible_calls = [
|
||||
tool_call
|
||||
for tool_call in tool_calls
|
||||
if not is_internal_tool_name(_serialized_tool_name(tool_call))
|
||||
]
|
||||
if not visible_calls:
|
||||
continue
|
||||
if len(visible_calls) != len(tool_calls):
|
||||
message = {**message, "tool_calls": visible_calls}
|
||||
elif kind in {"tool", "toolmessage"}:
|
||||
call_id = str(message.get("tool_call_id") or "")
|
||||
if is_internal_tool_name(message.get("name")) or call_id in internal_ids:
|
||||
internal_ids.discard(call_id)
|
||||
continue
|
||||
|
||||
projected.append(message)
|
||||
|
||||
return projected
|
||||
|
||||
|
||||
__all__ = [
|
||||
"INTERNAL_TOOL_NAMES",
|
||||
"internal_tool_call_ids",
|
||||
"is_internal_tool_name",
|
||||
"project_outward_messages",
|
||||
]
|
||||
Reference in new issue
Block a user