sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -21,6 +21,7 @@ import re
import shutil
import subprocess
import tempfile
import threading
from pathlib import Path
from deerflow.config.app_config import get_app_config
@@ -55,17 +56,46 @@ _LIBREOFFICE_TARGETS = {
}
_LIBREOFFICE_TIMEOUT_SECONDS = 120
# Files larger than this threshold are converted in a background thread.
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
# scheduling overhead for them.
_ASYNC_THRESHOLD_BYTES = 1 * 1024 * 1024 # 1 MB
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
_CONVERSION_CONCURRENCY = 2
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
# If pymupdf4llm produces fewer characters *per page* than this threshold,
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
# Rationale: normal text PDFs yield 200-2000 chars/page; image-based PDFs
# yield close to 0. 50 chars/page gives a wide safety margin.
# Falls back to absolute 200-char check when page count is unavailable.
_MIN_CHARS_PER_PAGE = 50
_MIN_CHARS_PER_PAGE = 50
def markdown_sidecar_path(file_path: Path) -> Path:
"""Return the collision-free Markdown companion path for a document.
Keeping the original extension (``report.docx.md``) avoids the historical
collision where ``report.docx`` and ``report.pdf`` both overwrote
``report.md``.
"""
return file_path.with_name(f"{file_path.name}.md")
def find_markdown_sidecar(file_path: Path) -> Path | None:
"""Find a new-style sidecar, falling back to the legacy ``<stem>.md``."""
# Only formats handled by the conversion pipeline can own a generated
# Markdown companion. Without this guard, unrelated uploads such as
# ``report.txt`` or ``report.jpg`` incorrectly claimed a real user upload
# named ``report.md`` as their legacy sidecar.
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
current = markdown_sidecar_path(file_path)
if current.is_file():
return current
legacy = file_path.with_suffix(".md")
if legacy != file_path and legacy.is_file():
return legacy
return None
def _pymupdf_output_too_sparse(text: str, file_path: Path) -> bool:
@@ -187,6 +217,17 @@ def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
return _convert_with_markitdown(converted_path)
def _bound_converted_markdown(text: str) -> str:
"""Apply a non-disableable cap before converted text is persisted or returned."""
if len(text) <= MAX_CONVERTED_MARKDOWN_CHARS:
return text
marker = (
"\n\n<!-- deerflow-conversion-truncated: extracted text exceeded "
f"{MAX_CONVERTED_MARKDOWN_CHARS} characters; use the original file for targeted inspection -->\n"
)
return text[: MAX_CONVERTED_MARKDOWN_CHARS - len(marker)] + marker
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
@@ -194,6 +235,12 @@ def _do_convert(file_path: Path, pdf_converter: str) -> str:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
with _CONVERSION_SEMAPHORE:
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
"""Parser implementation; callers must use :func:`_do_convert`."""
is_pdf = file_path.suffix.lower() == ".pdf"
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
@@ -239,14 +286,12 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""
try:
pdf_converter = _get_pdf_converter()
file_size = file_path.stat().st_size
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
# large factor even for a small upload. Always keep them off the event
# loop; the synchronous worker has a bounded global concurrency gate.
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
if file_size > _ASYNC_THRESHOLD_BYTES:
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
else:
text = _do_convert(file_path, pdf_converter)
md_path = file_path.with_suffix(".md")
md_path = markdown_sidecar_path(file_path)
md_path.write_text(text, encoding="utf-8")
logger.info("Converted %s to markdown: %s (%d chars)", file_path.name, md_path.name, len(text))
@@ -16,14 +16,34 @@ class InlineThinkTagFilter:
not safe: it can leak a partial tag or the model's reasoning into the
visible draft. This small state machine holds only the suffix that might
still form a tag and emits all confirmed answer text immediately.
``leading_only=True`` reserves only one optional response-leading envelope
for reasoning. It is suitable for exact payloads such as file bodies,
where a later literal ``<think>`` must remain ordinary content.
"""
_OPEN_TAG = "<think>"
_CLOSE_TAG = "</think>"
def __init__(self) -> None:
def __init__(
self,
*,
allow_implicit_close: bool = False,
leading_only: bool = False,
) -> None:
self._inside_think = False
self._pending = ""
self._channel_decided = not allow_implicit_close
# File-body transports need a stricter interpretation than ordinary
# chat rendering: only one optional reasoning envelope at the start of
# the response is protocol metadata. Once file bytes begin, a literal
# ``<think>`` is user content and must never be rewritten.
self._leading_only = leading_only
self._leading_phase = (
"implicit_probe"
if leading_only and allow_implicit_close
else ("probe" if leading_only else "disabled")
)
@staticmethod
def _possible_tag_prefix_size(content: str, tag: str) -> int:
@@ -39,11 +59,31 @@ class InlineThinkTagFilter:
if not chunk:
return "", ""
if self._leading_only:
return self._push_leading_parts(chunk)
remaining = self._pending + chunk
self._pending = ""
visible: list[str] = []
thinking: list[str] = []
if not self._channel_decided:
lowered = remaining.lower()
open_at = lowered.find(self._OPEN_TAG)
close_at = lowered.find(self._CLOSE_TAG)
if close_at >= 0 and (open_at < 0 or close_at < open_at):
# Some templates inject the opening tag outside returned
# content. Keep the ambiguous prefix private until its close
# marker arrives, even when that marker spans chunks.
thinking.append(remaining[:close_at])
remaining = remaining[close_at + len(self._CLOSE_TAG) :]
self._channel_decided = True
elif open_at >= 0:
self._channel_decided = True
else:
self._pending = remaining
return "", ""
while remaining:
tag = self._CLOSE_TAG if self._inside_think else self._OPEN_TAG
marker_at = remaining.lower().find(tag)
@@ -61,12 +101,116 @@ class InlineThinkTagFilter:
return "".join(visible), "".join(thinking)
def _push_leading_parts(self, chunk: str) -> tuple[str, str]:
"""Route only a leading ``<think>`` envelope as provider reasoning.
This mode is intended for protocol payloads such as streamed file
bodies. It waits only while the response prefix could still become an
opening tag, supports tags split across arbitrary chunks, and locks to
the visible channel immediately after the closing tag or the first
confirmed file byte. Later literal tags are therefore preserved.
"""
remaining = self._pending + chunk
self._pending = ""
if self._leading_phase == "visible":
return remaining, ""
if self._leading_phase in {"probe", "implicit_probe"}:
candidate = remaining.lstrip()
lowered = candidate.lower()
if lowered.startswith(self._OPEN_TAG):
# Whitespace before a leading reasoning envelope belongs to
# the provider's presentation, not to the prepared file.
remaining = candidate[len(self._OPEN_TAG) :]
self._leading_phase = "thinking"
self._inside_think = True
elif self._OPEN_TAG.startswith(lowered):
# Empty/whitespace-only prefixes and partial tags remain
# ambiguous until the next provider chunk arrives.
self._pending = remaining
return "", ""
elif self._leading_phase == "implicit_probe":
close_at = remaining.lower().find(self._CLOSE_TAG)
if close_at >= 0:
# Some chat templates inject ``<think>`` outside the
# returned token stream. This mode is enabled only when
# the immediately preceding bridge-selection response
# demonstrated that exact close-only convention. Until
# the closing marker arrives the prefix is intentionally
# buffered: ordinary file text and implicit reasoning are
# otherwise indistinguishable.
thinking = remaining[:close_at]
visible = remaining[close_at + len(self._CLOSE_TAG) :]
self._leading_phase = "visible"
return visible, thinking
self._pending = remaining
return "", ""
else:
self._leading_phase = "visible"
return remaining, ""
marker_at = remaining.lower().find(self._CLOSE_TAG)
if marker_at >= 0:
thinking = remaining[:marker_at]
visible = remaining[marker_at + len(self._CLOSE_TAG) :]
self._leading_phase = "visible"
self._inside_think = False
return visible, thinking
suffix_size = self._possible_tag_prefix_size(remaining, self._CLOSE_TAG)
thinking = remaining[:-suffix_size] if suffix_size else remaining
self._pending = remaining[-suffix_size:] if suffix_size else ""
return "", thinking
def push(self, chunk: str) -> str:
"""Compatibility helper returning only confirmed visible text."""
return self.push_parts(chunk)[0]
def confirm_visible_channel(self) -> str:
"""Stop close-only probing and return any buffered visible prelude.
A provider that emits a dedicated reasoning field has already told us
how to classify its trace. Its ordinary ``content`` channel can then
stream immediately even when the model supports thinking.
"""
if self._leading_only:
if self._leading_phase != "probe":
if self._leading_phase != "implicit_probe":
return ""
self._leading_phase = "visible"
pending = self._pending
self._pending = ""
return pending
if self._channel_decided:
return ""
self._channel_decided = True
pending = self._pending
self._pending = ""
return pending
def finish_parts(self) -> tuple[str, str]:
"""Flush the remaining visible or thinking text at end of stream."""
if self._leading_only:
tail = self._pending
self._pending = ""
if self._leading_phase == "implicit_probe":
# No close-only marker ever arrived. The buffered bytes are a
# normal file body, not reasoning; preserve them exactly.
self._leading_phase = "visible"
return tail, ""
if self._leading_phase == "thinking":
self._inside_think = False
self._leading_phase = "visible"
return "", tail
self._leading_phase = "visible"
return tail, ""
if not self._channel_decided:
tail = self._pending
self._pending = ""
self._channel_decided = True
return tail, ""
if self._inside_think:
thinking = self._pending
self._pending = ""
@@ -80,6 +224,14 @@ class InlineThinkTagFilter:
"""Compatibility helper returning only the visible stream tail."""
return self.finish_parts()[0]
@property
def has_unclosed_thinking(self) -> bool:
"""Whether the stream is currently inside a reasoning envelope."""
if self._leading_only:
return self._leading_phase == "thinking"
return self._inside_think
def strip_inline_think_tags(content: str) -> str:
"""Return complete text with inline model-thinking blocks removed."""
@@ -0,0 +1,97 @@
"""Visibility rules for transport-only tools used inside the agent graph.
The graph/checkpoint must retain these calls because they are part of the
model's causal history. Public serializers use the projection helpers below
without mutating that history.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any
INTERNAL_TOOL_NAMES = frozenset({"begin_write_file"})
def is_internal_tool_name(name: object) -> bool:
"""Return whether *name* is an exact match for a transport-only tool."""
return isinstance(name, str) and name in INTERNAL_TOOL_NAMES
def _serialized_tool_name(tool_call: object) -> object:
if not isinstance(tool_call, Mapping):
return None
name = tool_call.get("name")
if name is not None:
return name
function = tool_call.get("function")
return function.get("name") if isinstance(function, Mapping) else None
def internal_tool_call_ids(messages: Iterable[object]) -> set[str]:
"""Collect ids belonging to exact internal calls in serialized messages."""
ids: set[str] = set()
for message in messages:
if not isinstance(message, Mapping):
continue
tool_calls = message.get("tool_calls")
if not isinstance(tool_calls, (list, tuple)):
continue
for tool_call in tool_calls:
if not is_internal_tool_name(_serialized_tool_name(tool_call)):
continue
call_id = tool_call.get("id") if isinstance(tool_call, Mapping) else None
if call_id:
ids.add(str(call_id))
return ids
def project_outward_messages(
messages: Iterable[Any],
*,
known_internal_tool_call_ids: Iterable[str] = (),
) -> list[Any]:
"""Remove transport-only call/result pairs from serialized messages."""
materialized = list(messages)
internal_ids = {str(call_id) for call_id in known_internal_tool_call_ids if call_id}
projected: list[Any] = []
for message in materialized:
if not isinstance(message, Mapping):
projected.append(message)
continue
kind = str(message.get("type") or message.get("role") or "").lower()
if kind in {"ai", "assistant", "aimessage", "aimessagechunk"}:
tool_calls = message.get("tool_calls")
if isinstance(tool_calls, (list, tuple)) and tool_calls:
internal_ids.update(internal_tool_call_ids([message]))
visible_calls = [
tool_call
for tool_call in tool_calls
if not is_internal_tool_name(_serialized_tool_name(tool_call))
]
if not visible_calls:
continue
if len(visible_calls) != len(tool_calls):
message = {**message, "tool_calls": visible_calls}
elif kind in {"tool", "toolmessage"}:
call_id = str(message.get("tool_call_id") or "")
if is_internal_tool_name(message.get("name")) or call_id in internal_ids:
internal_ids.discard(call_id)
continue
projected.append(message)
return projected
__all__ = [
"INTERNAL_TOOL_NAMES",
"internal_tool_call_ids",
"is_internal_tool_name",
"project_outward_messages",
]