feat: standardize Word export and Office conversion
This commit is contained in:
1 parent
4ce2f3a502
commit
485a8fa9d9
12 files changed
+410
-60
No files matched your search
+104
-15
@@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
|
||||
No FastAPI or HTTP dependencies — pure utility functions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File extensions that should be converted to markdown
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".doc",
|
||||
".docx",
|
||||
}
|
||||
".docx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
}
|
||||
|
||||
# MarkItDown reads modern OOXML files directly. LibreOffice is used as a
|
||||
# compatibility bridge for legacy binary Office files and OpenDocument files,
|
||||
# which are common in offline/intranet deployments but are not handled
|
||||
# consistently by the pure-Python readers.
|
||||
_LIBREOFFICE_TARGETS = {
|
||||
".doc": ".docx",
|
||||
".ppt": ".pptx",
|
||||
".xls": ".xlsx",
|
||||
".odt": ".docx",
|
||||
".odp": ".pptx",
|
||||
".ods": ".xlsx",
|
||||
}
|
||||
_LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
|
||||
# Files larger than this threshold are converted in a background thread.
|
||||
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
|
||||
@@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
"""Convert any supported file to markdown text using MarkItDown."""
|
||||
from markitdown import MarkItDown
|
||||
|
||||
md = MarkItDown()
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
|
||||
"""Normalize a legacy Office/OpenDocument file, then extract its text.
|
||||
|
||||
Each conversion gets an isolated LibreOffice user profile. This avoids
|
||||
the profile-lock conflict that otherwise appears when several uploads are
|
||||
converted concurrently in the same sandbox container.
|
||||
"""
|
||||
target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower())
|
||||
if target_suffix is None:
|
||||
return None
|
||||
|
||||
executable = shutil.which("soffice") or shutil.which("libreoffice")
|
||||
if executable is None:
|
||||
logger.warning(
|
||||
"LibreOffice is unavailable; trying MarkItDown directly for %s",
|
||||
file_path.name,
|
||||
)
|
||||
return None
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary:
|
||||
temp_root = Path(temporary).resolve()
|
||||
output_dir = temp_root / "output"
|
||||
profile_dir = temp_root / "profile"
|
||||
output_dir.mkdir()
|
||||
profile_dir.mkdir()
|
||||
command = [
|
||||
executable,
|
||||
"--headless",
|
||||
"--nologo",
|
||||
"--nodefault",
|
||||
"--nolockcheck",
|
||||
"--nofirststartwizard",
|
||||
f"-env:UserInstallation={profile_dir.as_uri()}",
|
||||
"--convert-to",
|
||||
target_suffix.lstrip("."),
|
||||
"--outdir",
|
||||
str(output_dir),
|
||||
str(file_path.resolve()),
|
||||
]
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_LIBREOFFICE_TIMEOUT_SECONDS,
|
||||
env={**os.environ, "HOME": str(temp_root)},
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
logger.exception("LibreOffice failed to normalize %s", file_path.name)
|
||||
return None
|
||||
|
||||
converted_path = output_dir / f"{file_path.stem}{target_suffix}"
|
||||
if completed.returncode != 0 or not converted_path.is_file():
|
||||
logger.error(
|
||||
"LibreOffice could not normalize %s (exit=%s, stderr=%s)",
|
||||
file_path.name,
|
||||
completed.returncode,
|
||||
completed.stderr.strip()[-1000:],
|
||||
)
|
||||
return None
|
||||
return _convert_with_markitdown(converted_path)
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
|
||||
if normalized_office_text is not None:
|
||||
return normalized_office_text
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
# Try pymupdf4llm first (auto or explicit)
|
||||
pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path)
|
||||
|
||||
|
||||
Reference in new issue
Block a user