sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+57
-12
@@ -21,6 +21,7 @@ import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
@@ -55,17 +56,46 @@ _LIBREOFFICE_TARGETS = {
|
||||
}
|
||||
_LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
|
||||
# Files larger than this threshold are converted in a background thread.
|
||||
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
|
||||
# scheduling overhead for them.
|
||||
_ASYNC_THRESHOLD_BYTES = 1 * 1024 * 1024 # 1 MB
|
||||
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
|
||||
_CONVERSION_CONCURRENCY = 2
|
||||
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
|
||||
|
||||
# If pymupdf4llm produces fewer characters *per page* than this threshold,
|
||||
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
|
||||
# Rationale: normal text PDFs yield 200-2000 chars/page; image-based PDFs
|
||||
# yield close to 0. 50 chars/page gives a wide safety margin.
|
||||
# Falls back to absolute 200-char check when page count is unavailable.
|
||||
_MIN_CHARS_PER_PAGE = 50
|
||||
_MIN_CHARS_PER_PAGE = 50
|
||||
|
||||
|
||||
def markdown_sidecar_path(file_path: Path) -> Path:
|
||||
"""Return the collision-free Markdown companion path for a document.
|
||||
|
||||
Keeping the original extension (``report.docx.md``) avoids the historical
|
||||
collision where ``report.docx`` and ``report.pdf`` both overwrote
|
||||
``report.md``.
|
||||
"""
|
||||
|
||||
return file_path.with_name(f"{file_path.name}.md")
|
||||
|
||||
|
||||
def find_markdown_sidecar(file_path: Path) -> Path | None:
|
||||
"""Find a new-style sidecar, falling back to the legacy ``<stem>.md``."""
|
||||
|
||||
# Only formats handled by the conversion pipeline can own a generated
|
||||
# Markdown companion. Without this guard, unrelated uploads such as
|
||||
# ``report.txt`` or ``report.jpg`` incorrectly claimed a real user upload
|
||||
# named ``report.md`` as their legacy sidecar.
|
||||
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
|
||||
return None
|
||||
|
||||
current = markdown_sidecar_path(file_path)
|
||||
if current.is_file():
|
||||
return current
|
||||
legacy = file_path.with_suffix(".md")
|
||||
if legacy != file_path and legacy.is_file():
|
||||
return legacy
|
||||
return None
|
||||
|
||||
|
||||
def _pymupdf_output_too_sparse(text: str, file_path: Path) -> bool:
|
||||
@@ -187,6 +217,17 @@ def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
|
||||
return _convert_with_markitdown(converted_path)
|
||||
|
||||
|
||||
def _bound_converted_markdown(text: str) -> str:
|
||||
"""Apply a non-disableable cap before converted text is persisted or returned."""
|
||||
if len(text) <= MAX_CONVERTED_MARKDOWN_CHARS:
|
||||
return text
|
||||
marker = (
|
||||
"\n\n<!-- deerflow-conversion-truncated: extracted text exceeded "
|
||||
f"{MAX_CONVERTED_MARKDOWN_CHARS} characters; use the original file for targeted inspection -->\n"
|
||||
)
|
||||
return text[: MAX_CONVERTED_MARKDOWN_CHARS - len(marker)] + marker
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
|
||||
@@ -194,6 +235,12 @@ def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
with _CONVERSION_SEMAPHORE:
|
||||
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
|
||||
|
||||
|
||||
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Parser implementation; callers must use :func:`_do_convert`."""
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
|
||||
@@ -239,14 +286,12 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
"""
|
||||
try:
|
||||
pdf_converter = _get_pdf_converter()
|
||||
file_size = file_path.stat().st_size
|
||||
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
|
||||
# large factor even for a small upload. Always keep them off the event
|
||||
# loop; the synchronous worker has a bounded global concurrency gate.
|
||||
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
|
||||
|
||||
if file_size > _ASYNC_THRESHOLD_BYTES:
|
||||
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
|
||||
else:
|
||||
text = _do_convert(file_path, pdf_converter)
|
||||
|
||||
md_path = file_path.with_suffix(".md")
|
||||
md_path = markdown_sidecar_path(file_path)
|
||||
md_path.write_text(text, encoding="utf-8")
|
||||
|
||||
logger.info("Converted %s to markdown: %s (%d chars)", file_path.name, md_path.name, len(text))
|
||||
|
||||
Reference in new issue
Block a user