sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -8,23 +8,28 @@ PDF conversion strategy (auto mode):
total when page count is unavailable), treat as image-based and fall back to MarkItDown.
3. If pymupdf4llm is not installed, use MarkItDown directly (existing behaviour).
Large files (> ASYNC_THRESHOLD_BYTES) are converted in a thread pool via
asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
Parsing runs in a bounded pool of spawned processes so CPU-heavy document
readers cannot hold the gateway's GIL. Async callers wait and write sidecars in
a dedicated thread pool, leaving the event loop and default executor available.
No FastAPI or HTTP dependencies — pure utility functions.
"""
import asyncio
import atexit
import logging
import multiprocessing
import os
import re
import shutil
import subprocess
import tempfile
import threading
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures.process import BrokenProcessPool
from pathlib import Path
from deerflow.config.app_config import get_app_config
from deerflow.utils.file_io import FileIOPool
logger = logging.getLogger(__name__)
@@ -59,6 +64,59 @@ _LIBREOFFICE_TIMEOUT_SECONDS = 120
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
_CONVERSION_CONCURRENCY = 2
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
_CONVERSION_IO = FileIOPool(_CONVERSION_CONCURRENCY, _CONVERSION_CONCURRENCY, "document-conversion")
_PROCESS_POOL_LOCK = threading.Lock()
_PROCESS_POOL: ProcessPoolExecutor | None = None
def _get_conversion_process_pool() -> ProcessPoolExecutor:
"""Lazily start isolated readers without forking a multithreaded gateway."""
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
if _PROCESS_POOL is None:
_PROCESS_POOL = ProcessPoolExecutor(
max_workers=_CONVERSION_CONCURRENCY,
mp_context=multiprocessing.get_context("spawn"),
)
return _PROCESS_POOL
def shutdown_conversion_workers(*, wait: bool = True) -> None:
"""Release document reader processes; a later request can start a new pool."""
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
pool, _PROCESS_POOL = _PROCESS_POOL, None
if pool is not None:
pool.shutdown(wait=wait, cancel_futures=True)
def _discard_broken_process_pool(pool: ProcessPoolExecutor) -> None:
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
if _PROCESS_POOL is not pool:
return
_PROCESS_POOL = None
pool.shutdown(wait=False, cancel_futures=True)
def _convert_document_worker(file_path: str, pdf_converter: str) -> str:
"""Child entry point: only the path and converter cross the process boundary."""
return _bound_converted_markdown(_do_convert_unbounded(Path(file_path), pdf_converter))
def _run_conversion_in_process(file_path: Path, pdf_converter: str) -> str:
pool = _get_conversion_process_pool()
try:
return pool.submit(_convert_document_worker, str(file_path), pdf_converter).result()
except BrokenProcessPool:
# Do not retry a document that may have crashed a native parser. The
# existing public API reports conversion failure; the next request gets
# fresh workers instead of inheriting a permanently unusable executor.
_discard_broken_process_pool(pool)
raise
atexit.register(shutdown_conversion_workers)
# If pymupdf4llm produces fewer characters *per page* than this threshold,
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
@@ -229,14 +287,14 @@ def _bound_converted_markdown(text: str) -> str:
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
"""Synchronous conversion with a shared limit for sync and async callers.
Args:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
with _CONVERSION_SEMAPHORE:
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
return _run_conversion_in_process(file_path, pdf_converter)
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
@@ -271,25 +329,15 @@ def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
return _convert_with_markitdown(file_path)
async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""Convert a supported document file to Markdown.
def convert_file_to_markdown_sync(file_path: Path) -> Path | None:
"""Convert and write a sidecar inside an existing file worker.
PDF files are handled with a two-converter strategy (see module docstring).
Large files (> 1 MB) are offloaded to a thread pool to avoid blocking the
event loop.
Args:
file_path: Path to the file to convert.
Returns:
Path to the generated .md file, or None if conversion failed.
Callers must keep this synchronous parsing and disk work off an event loop.
The shared conversion gate also applies to upload workers calling directly.
"""
try:
pdf_converter = _get_pdf_converter()
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
# large factor even for a small upload. Always keep them off the event
# loop; the synchronous worker has a bounded global concurrency gate.
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
text = _do_convert(file_path, pdf_converter)
md_path = markdown_sidecar_path(file_path)
md_path.write_text(text, encoding="utf-8")
@@ -299,6 +347,19 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
except Exception as e:
logger.error("Failed to convert %s to markdown: %s", file_path.name, e)
return None
async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""Convert and persist Markdown without blocking other requests.
Admission waits on the event loop, never inside the default executor.
Cancellation waits for the worker so callers can safely remove its input.
"""
return await _CONVERSION_IO.run_until_complete(convert_file_to_markdown_sync, file_path)
def _extract_file_to_markdown_sync(file_path: Path) -> str:
return _do_convert(file_path, _get_pdf_converter())
async def extract_file_to_markdown(file_path: Path) -> str | None:
@@ -313,7 +374,7 @@ async def extract_file_to_markdown(file_path: Path) -> str | None:
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
return await _CONVERSION_IO.run_until_complete(_extract_file_to_markdown_sync, file_path)
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None