sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+82
-21
@@ -8,23 +8,28 @@ PDF conversion strategy (auto mode):
|
||||
total when page count is unavailable), treat as image-based and fall back to MarkItDown.
|
||||
3. If pymupdf4llm is not installed, use MarkItDown directly (existing behaviour).
|
||||
|
||||
Large files (> ASYNC_THRESHOLD_BYTES) are converted in a thread pool via
|
||||
asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
|
||||
Parsing runs in a bounded pool of spawned processes so CPU-heavy document
|
||||
readers cannot hold the gateway's GIL. Async callers wait and write sidecars in
|
||||
a dedicated thread pool, leaving the event loop and default executor available.
|
||||
|
||||
No FastAPI or HTTP dependencies — pure utility functions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.utils.file_io import FileIOPool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -59,6 +64,59 @@ _LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
|
||||
_CONVERSION_CONCURRENCY = 2
|
||||
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
|
||||
_CONVERSION_IO = FileIOPool(_CONVERSION_CONCURRENCY, _CONVERSION_CONCURRENCY, "document-conversion")
|
||||
_PROCESS_POOL_LOCK = threading.Lock()
|
||||
_PROCESS_POOL: ProcessPoolExecutor | None = None
|
||||
|
||||
|
||||
def _get_conversion_process_pool() -> ProcessPoolExecutor:
|
||||
"""Lazily start isolated readers without forking a multithreaded gateway."""
|
||||
global _PROCESS_POOL
|
||||
with _PROCESS_POOL_LOCK:
|
||||
if _PROCESS_POOL is None:
|
||||
_PROCESS_POOL = ProcessPoolExecutor(
|
||||
max_workers=_CONVERSION_CONCURRENCY,
|
||||
mp_context=multiprocessing.get_context("spawn"),
|
||||
)
|
||||
return _PROCESS_POOL
|
||||
|
||||
|
||||
def shutdown_conversion_workers(*, wait: bool = True) -> None:
|
||||
"""Release document reader processes; a later request can start a new pool."""
|
||||
global _PROCESS_POOL
|
||||
with _PROCESS_POOL_LOCK:
|
||||
pool, _PROCESS_POOL = _PROCESS_POOL, None
|
||||
if pool is not None:
|
||||
pool.shutdown(wait=wait, cancel_futures=True)
|
||||
|
||||
|
||||
def _discard_broken_process_pool(pool: ProcessPoolExecutor) -> None:
|
||||
global _PROCESS_POOL
|
||||
with _PROCESS_POOL_LOCK:
|
||||
if _PROCESS_POOL is not pool:
|
||||
return
|
||||
_PROCESS_POOL = None
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
||||
def _convert_document_worker(file_path: str, pdf_converter: str) -> str:
|
||||
"""Child entry point: only the path and converter cross the process boundary."""
|
||||
return _bound_converted_markdown(_do_convert_unbounded(Path(file_path), pdf_converter))
|
||||
|
||||
|
||||
def _run_conversion_in_process(file_path: Path, pdf_converter: str) -> str:
|
||||
pool = _get_conversion_process_pool()
|
||||
try:
|
||||
return pool.submit(_convert_document_worker, str(file_path), pdf_converter).result()
|
||||
except BrokenProcessPool:
|
||||
# Do not retry a document that may have crashed a native parser. The
|
||||
# existing public API reports conversion failure; the next request gets
|
||||
# fresh workers instead of inheriting a permanently unusable executor.
|
||||
_discard_broken_process_pool(pool)
|
||||
raise
|
||||
|
||||
|
||||
atexit.register(shutdown_conversion_workers)
|
||||
|
||||
# If pymupdf4llm produces fewer characters *per page* than this threshold,
|
||||
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
|
||||
@@ -229,14 +287,14 @@ def _bound_converted_markdown(text: str) -> str:
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
"""Synchronous conversion with a shared limit for sync and async callers.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
with _CONVERSION_SEMAPHORE:
|
||||
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
|
||||
return _run_conversion_in_process(file_path, pdf_converter)
|
||||
|
||||
|
||||
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
|
||||
@@ -271,25 +329,15 @@ def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
|
||||
return _convert_with_markitdown(file_path)
|
||||
|
||||
|
||||
async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
"""Convert a supported document file to Markdown.
|
||||
def convert_file_to_markdown_sync(file_path: Path) -> Path | None:
|
||||
"""Convert and write a sidecar inside an existing file worker.
|
||||
|
||||
PDF files are handled with a two-converter strategy (see module docstring).
|
||||
Large files (> 1 MB) are offloaded to a thread pool to avoid blocking the
|
||||
event loop.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to convert.
|
||||
|
||||
Returns:
|
||||
Path to the generated .md file, or None if conversion failed.
|
||||
Callers must keep this synchronous parsing and disk work off an event loop.
|
||||
The shared conversion gate also applies to upload workers calling directly.
|
||||
"""
|
||||
try:
|
||||
pdf_converter = _get_pdf_converter()
|
||||
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
|
||||
# large factor even for a small upload. Always keep them off the event
|
||||
# loop; the synchronous worker has a bounded global concurrency gate.
|
||||
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
|
||||
text = _do_convert(file_path, pdf_converter)
|
||||
|
||||
md_path = markdown_sidecar_path(file_path)
|
||||
md_path.write_text(text, encoding="utf-8")
|
||||
@@ -299,6 +347,19 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
except Exception as e:
|
||||
logger.error("Failed to convert %s to markdown: %s", file_path.name, e)
|
||||
return None
|
||||
|
||||
|
||||
async def convert_file_to_markdown(file_path: Path) -> Path | None:
|
||||
"""Convert and persist Markdown without blocking other requests.
|
||||
|
||||
Admission waits on the event loop, never inside the default executor.
|
||||
Cancellation waits for the worker so callers can safely remove its input.
|
||||
"""
|
||||
return await _CONVERSION_IO.run_until_complete(convert_file_to_markdown_sync, file_path)
|
||||
|
||||
|
||||
def _extract_file_to_markdown_sync(file_path: Path) -> str:
|
||||
return _do_convert(file_path, _get_pdf_converter())
|
||||
|
||||
|
||||
async def extract_file_to_markdown(file_path: Path) -> str | None:
|
||||
@@ -313,7 +374,7 @@ async def extract_file_to_markdown(file_path: Path) -> str | None:
|
||||
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
|
||||
return None
|
||||
try:
|
||||
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
|
||||
return await _CONVERSION_IO.run_until_complete(_extract_file_to_markdown_sync, file_path)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
|
||||
return None
|
||||
|
||||
Reference in new issue
Block a user