feat: standardize Word export and Office conversion

This commit is contained in:
吉鑫升 committed 2026-08-27 00:09:53 +08:00
1 parent 4ce2f3a502
commit 485a8fa9d9
12 files changed
+410 -60

No files matched your search

@@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
No FastAPI or HTTP dependencies — pure utility functions.
"""
import asyncio
import logging
import re
from pathlib import Path
import asyncio
import logging
import os
import re
import shutil
import subprocess
import tempfile
from pathlib import Path
from deerflow.config.app_config import get_app_config
logger = logging.getLogger(__name__)
# File extensions that should be converted to markdown
CONVERTIBLE_EXTENSIONS = {
CONVERTIBLE_EXTENSIONS = {
".pdf",
".ppt",
".pptx",
".xls",
".xlsx",
".doc",
".docx",
}
".docx",
".odt",
".ods",
".odp",
}
# MarkItDown reads modern OOXML files directly. LibreOffice is used as a
# compatibility bridge for legacy binary Office files and OpenDocument files,
# which are common in offline/intranet deployments but are not handled
# consistently by the pure-Python readers.
_LIBREOFFICE_TARGETS = {
".doc": ".docx",
".ppt": ".pptx",
".xls": ".xlsx",
".odt": ".docx",
".odp": ".pptx",
".ods": ".xlsx",
}
_LIBREOFFICE_TIMEOUT_SECONDS = 120
# Files larger than this threshold are converted in a background thread.
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
@@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None:
return None
def _convert_with_markitdown(file_path: Path) -> str:
def _convert_with_markitdown(file_path: Path) -> str:
"""Convert any supported file to markdown text using MarkItDown."""
from markitdown import MarkItDown
md = MarkItDown()
return md.convert(str(file_path)).text_content
def _do_convert(file_path: Path, pdf_converter: str) -> str:
return md.convert(str(file_path)).text_content
def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
"""Normalize a legacy Office/OpenDocument file, then extract its text.
Each conversion gets an isolated LibreOffice user profile. This avoids
the profile-lock conflict that otherwise appears when several uploads are
converted concurrently in the same sandbox container.
"""
target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower())
if target_suffix is None:
return None
executable = shutil.which("soffice") or shutil.which("libreoffice")
if executable is None:
logger.warning(
"LibreOffice is unavailable; trying MarkItDown directly for %s",
file_path.name,
)
return None
with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary:
temp_root = Path(temporary).resolve()
output_dir = temp_root / "output"
profile_dir = temp_root / "profile"
output_dir.mkdir()
profile_dir.mkdir()
command = [
executable,
"--headless",
"--nologo",
"--nodefault",
"--nolockcheck",
"--nofirststartwizard",
f"-env:UserInstallation={profile_dir.as_uri()}",
"--convert-to",
target_suffix.lstrip("."),
"--outdir",
str(output_dir),
str(file_path.resolve()),
]
try:
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
timeout=_LIBREOFFICE_TIMEOUT_SECONDS,
env={**os.environ, "HOME": str(temp_root)},
)
except (OSError, subprocess.TimeoutExpired):
logger.exception("LibreOffice failed to normalize %s", file_path.name)
return None
converted_path = output_dir / f"{file_path.stem}{target_suffix}"
if completed.returncode != 0 or not converted_path.is_file():
logger.error(
"LibreOffice could not normalize %s (exit=%s, stderr=%s)",
file_path.name,
completed.returncode,
completed.stderr.strip()[-1000:],
)
return None
return _convert_with_markitdown(converted_path)
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
Args:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
is_pdf = file_path.suffix.lower() == ".pdf"
if is_pdf and pdf_converter != "markitdown":
is_pdf = file_path.suffix.lower() == ".pdf"
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
if normalized_office_text is not None:
return normalized_office_text
if is_pdf and pdf_converter != "markitdown":
# Try pymupdf4llm first (auto or explicit)
pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path)