sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -21,6 +21,7 @@ import re
import shutil
import subprocess
import tempfile
import threading
from pathlib import Path
from deerflow.config.app_config import get_app_config
@@ -55,17 +56,46 @@ _LIBREOFFICE_TARGETS = {
}
_LIBREOFFICE_TIMEOUT_SECONDS = 120
# Files larger than this threshold are converted in a background thread.
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
# scheduling overhead for them.
_ASYNC_THRESHOLD_BYTES = 1 * 1024 * 1024 # 1 MB
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
_CONVERSION_CONCURRENCY = 2
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
# If pymupdf4llm produces fewer characters *per page* than this threshold,
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
# Rationale: normal text PDFs yield 200-2000 chars/page; image-based PDFs
# yield close to 0. 50 chars/page gives a wide safety margin.
# Falls back to absolute 200-char check when page count is unavailable.
_MIN_CHARS_PER_PAGE = 50
_MIN_CHARS_PER_PAGE = 50
def markdown_sidecar_path(file_path: Path) -> Path:
"""Return the collision-free Markdown companion path for a document.
Keeping the original extension (``report.docx.md``) avoids the historical
collision where ``report.docx`` and ``report.pdf`` both overwrote
``report.md``.
"""
return file_path.with_name(f"{file_path.name}.md")
def find_markdown_sidecar(file_path: Path) -> Path | None:
"""Find a new-style sidecar, falling back to the legacy ``<stem>.md``."""
# Only formats handled by the conversion pipeline can own a generated
# Markdown companion. Without this guard, unrelated uploads such as
# ``report.txt`` or ``report.jpg`` incorrectly claimed a real user upload
# named ``report.md`` as their legacy sidecar.
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
current = markdown_sidecar_path(file_path)
if current.is_file():
return current
legacy = file_path.with_suffix(".md")
if legacy != file_path and legacy.is_file():
return legacy
return None
def _pymupdf_output_too_sparse(text: str, file_path: Path) -> bool:
@@ -187,6 +217,17 @@ def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
return _convert_with_markitdown(converted_path)
def _bound_converted_markdown(text: str) -> str:
"""Apply a non-disableable cap before converted text is persisted or returned."""
if len(text) <= MAX_CONVERTED_MARKDOWN_CHARS:
return text
marker = (
"\n\n<!-- deerflow-conversion-truncated: extracted text exceeded "
f"{MAX_CONVERTED_MARKDOWN_CHARS} characters; use the original file for targeted inspection -->\n"
)
return text[: MAX_CONVERTED_MARKDOWN_CHARS - len(marker)] + marker
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
@@ -194,6 +235,12 @@ def _do_convert(file_path: Path, pdf_converter: str) -> str:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
with _CONVERSION_SEMAPHORE:
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
"""Parser implementation; callers must use :func:`_do_convert`."""
is_pdf = file_path.suffix.lower() == ".pdf"
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
@@ -239,14 +286,12 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""
try:
pdf_converter = _get_pdf_converter()
file_size = file_path.stat().st_size
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
# large factor even for a small upload. Always keep them off the event
# loop; the synchronous worker has a bounded global concurrency gate.
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
if file_size > _ASYNC_THRESHOLD_BYTES:
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
else:
text = _do_convert(file_path, pdf_converter)
md_path = file_path.with_suffix(".md")
md_path = markdown_sidecar_path(file_path)
md_path.write_text(text, encoding="utf-8")
logger.info("Converted %s to markdown: %s (%d chars)", file_path.name, md_path.name, len(text))