sync update from LAN repo (source master @ ca48c4ec)

This commit is contained in:
gozeros committed 2026-11-05 10:53:54 +08:00
1 parent 8bc8b0e082
commit ea9dcff9c0
873 files changed
+107866 -14011

No files matched your search

@@ -0,0 +1,43 @@
"""Shared, bounded initialization for async agent and tool entry points."""
from __future__ import annotations
import asyncio
import contextvars
from collections.abc import Callable
from typing import Any
from deerflow.utils.file_io import FileIOPool
# Serialize cold imports/client setup without occupying the request loop or
# authentication's executor. Queued initializers consume no worker threads.
_INITIALIZATION_IO = FileIOPool(1, 1, "agent-init")
def _initialize_in_worker[Result](
function: Callable[..., Result], args: tuple[Any, ...], kwargs: dict[str, Any], loop: asyncio.AbstractEventLoop
) -> tuple[Result, contextvars.Context]:
from deerflow.mcp.cache import mcp_initialization_loop
# Synchronous construction may wait for MCP discovery, while its owning
# request loop remains free to run discovery and other requests.
with mcp_initialization_loop(loop):
result = function(*args, **kwargs)
return result, contextvars.copy_context()
async def run_background_init[Result](function: Callable[..., Result], /, *args: Any, **kwargs: Any) -> Result:
"""Run thread-safe initialization with the caller's MCP loop and context.
Cancellation drains already-submitted work before propagating and never
commits its context changes. Successful initialization writes ContextVars
back to this task so tool registries remain available during execution.
"""
result, initialized_context = await _INITIALIZATION_IO.run_until_complete(
_initialize_in_worker, function, args, kwargs, asyncio.get_running_loop()
)
# Ordinary to_thread only copies context into a worker. Tool discovery also
# sets or clears the deferred registry, which the calling task must inherit.
for variable, value in initialized_context.items():
variable.set(value)
return result
@@ -8,23 +8,28 @@ PDF conversion strategy (auto mode):
total when page count is unavailable), treat as image-based and fall back to MarkItDown.
3. If pymupdf4llm is not installed, use MarkItDown directly (existing behaviour).
Large files (> ASYNC_THRESHOLD_BYTES) are converted in a thread pool via
asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
Parsing runs in a bounded pool of spawned processes so CPU-heavy document
readers cannot hold the gateway's GIL. Async callers wait and write sidecars in
a dedicated thread pool, leaving the event loop and default executor available.
No FastAPI or HTTP dependencies — pure utility functions.
"""
import asyncio
import atexit
import logging
import multiprocessing
import os
import re
import shutil
import subprocess
import tempfile
import threading
from concurrent.futures import ProcessPoolExecutor
from concurrent.futures.process import BrokenProcessPool
from pathlib import Path
from deerflow.config.app_config import get_app_config
from deerflow.utils.file_io import FileIOPool
logger = logging.getLogger(__name__)
@@ -59,6 +64,59 @@ _LIBREOFFICE_TIMEOUT_SECONDS = 120
MAX_CONVERTED_MARKDOWN_CHARS = 2_000_000
_CONVERSION_CONCURRENCY = 2
_CONVERSION_SEMAPHORE = threading.BoundedSemaphore(_CONVERSION_CONCURRENCY)
_CONVERSION_IO = FileIOPool(_CONVERSION_CONCURRENCY, _CONVERSION_CONCURRENCY, "document-conversion")
_PROCESS_POOL_LOCK = threading.Lock()
_PROCESS_POOL: ProcessPoolExecutor | None = None
def _get_conversion_process_pool() -> ProcessPoolExecutor:
"""Lazily start isolated readers without forking a multithreaded gateway."""
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
if _PROCESS_POOL is None:
_PROCESS_POOL = ProcessPoolExecutor(
max_workers=_CONVERSION_CONCURRENCY,
mp_context=multiprocessing.get_context("spawn"),
)
return _PROCESS_POOL
def shutdown_conversion_workers(*, wait: bool = True) -> None:
"""Release document reader processes; a later request can start a new pool."""
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
pool, _PROCESS_POOL = _PROCESS_POOL, None
if pool is not None:
pool.shutdown(wait=wait, cancel_futures=True)
def _discard_broken_process_pool(pool: ProcessPoolExecutor) -> None:
global _PROCESS_POOL
with _PROCESS_POOL_LOCK:
if _PROCESS_POOL is not pool:
return
_PROCESS_POOL = None
pool.shutdown(wait=False, cancel_futures=True)
def _convert_document_worker(file_path: str, pdf_converter: str) -> str:
"""Child entry point: only the path and converter cross the process boundary."""
return _bound_converted_markdown(_do_convert_unbounded(Path(file_path), pdf_converter))
def _run_conversion_in_process(file_path: Path, pdf_converter: str) -> str:
pool = _get_conversion_process_pool()
try:
return pool.submit(_convert_document_worker, str(file_path), pdf_converter).result()
except BrokenProcessPool:
# Do not retry a document that may have crashed a native parser. The
# existing public API reports conversion failure; the next request gets
# fresh workers instead of inheriting a permanently unusable executor.
_discard_broken_process_pool(pool)
raise
atexit.register(shutdown_conversion_workers)
# If pymupdf4llm produces fewer characters *per page* than this threshold,
# the PDF is likely image-based or encrypted — fall back to MarkItDown.
@@ -229,14 +287,14 @@ def _bound_converted_markdown(text: str) -> str:
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
"""Synchronous conversion with a shared limit for sync and async callers.
Args:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
with _CONVERSION_SEMAPHORE:
return _bound_converted_markdown(_do_convert_unbounded(file_path, pdf_converter))
return _run_conversion_in_process(file_path, pdf_converter)
def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
@@ -271,25 +329,15 @@ def _do_convert_unbounded(file_path: Path, pdf_converter: str) -> str:
return _convert_with_markitdown(file_path)
async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""Convert a supported document file to Markdown.
def convert_file_to_markdown_sync(file_path: Path) -> Path | None:
"""Convert and write a sidecar inside an existing file worker.
PDF files are handled with a two-converter strategy (see module docstring).
Large files (> 1 MB) are offloaded to a thread pool to avoid blocking the
event loop.
Args:
file_path: Path to the file to convert.
Returns:
Path to the generated .md file, or None if conversion failed.
Callers must keep this synchronous parsing and disk work off an event loop.
The shared conversion gate also applies to upload workers calling directly.
"""
try:
pdf_converter = _get_pdf_converter()
# Office/PDF parsers can be CPU-heavy or expand compressed input by a
# large factor even for a small upload. Always keep them off the event
# loop; the synchronous worker has a bounded global concurrency gate.
text = await asyncio.to_thread(_do_convert, file_path, pdf_converter)
text = _do_convert(file_path, pdf_converter)
md_path = markdown_sidecar_path(file_path)
md_path.write_text(text, encoding="utf-8")
@@ -299,6 +347,19 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
except Exception as e:
logger.error("Failed to convert %s to markdown: %s", file_path.name, e)
return None
async def convert_file_to_markdown(file_path: Path) -> Path | None:
"""Convert and persist Markdown without blocking other requests.
Admission waits on the event loop, never inside the default executor.
Cancellation waits for the worker so callers can safely remove its input.
"""
return await _CONVERSION_IO.run_until_complete(convert_file_to_markdown_sync, file_path)
def _extract_file_to_markdown_sync(file_path: Path) -> str:
return _do_convert(file_path, _get_pdf_converter())
async def extract_file_to_markdown(file_path: Path) -> str | None:
@@ -313,7 +374,7 @@ async def extract_file_to_markdown(file_path: Path) -> str | None:
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
return await _CONVERSION_IO.run_until_complete(_extract_file_to_markdown_sync, file_path)
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None
@@ -0,0 +1,76 @@
"""Bounded file work, isolated from authentication's default executor."""
from __future__ import annotations
import asyncio
import contextvars
import logging
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from weakref import WeakKeyDictionary
logger = logging.getLogger(__name__)
class FileIOPool:
def __init__(self, workers: int, admitted: int, name: str):
self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix=name)
self._slots: WeakKeyDictionary = WeakKeyDictionary()
self._admitted = admitted
async def __call__(self, function, /, *args, **kwargs):
return await self._run(function, args, kwargs, wait_on_cancel=False, on_cancel=None)
async def run_until_complete(self, function, /, *args, on_cancel=None, **kwargs):
"""Keep a submitted worker alive until it finishes, even after cancellation.
Callers can safely close inputs or remove temporary directories once this
returns or raises. Cancellation while waiting for admission does not
submit work. Once submitted, ``on_cancel`` may request cooperative
cancellation; admission remains held until the actual worker exits.
"""
return await self._run(function, args, kwargs, wait_on_cancel=True, on_cancel=on_cancel)
async def _run(self, function, args, kwargs, *, wait_on_cancel, on_cancel):
loop = asyncio.get_running_loop()
semaphore = self._slots.setdefault(loop, asyncio.Semaphore(self._admitted))
await semaphore.acquire()
context = contextvars.copy_context()
try:
future = loop.run_in_executor(self._pool, context.run, partial(function, *args, **kwargs))
except BaseException:
semaphore.release()
raise
def finished(done):
semaphore.release()
if not done.cancelled():
done.exception()
future.add_done_callback(finished)
# Cancellation cannot release admission before the real disk job ends.
try:
return await asyncio.shield(future)
except asyncio.CancelledError:
if not wait_on_cancel:
raise
if on_cancel is not None:
try:
on_cancel()
except Exception:
logger.exception("File worker cancellation callback failed")
# The request owns the worker's inputs. Do not let its finally
# block clean those up while a thread still reads or writes them.
# Repeated Task.cancel() calls must not break that guarantee.
while not future.done():
try:
await asyncio.shield(future)
except asyncio.CancelledError:
continue
except BaseException:
break # finished() consumes the worker's exception.
raise
file_io = FileIOPool(4, 8, "artifact-files")
event_io = FileIOPool(2, 4, "event-spool")