feat: standardize Word export and Office conversion

This commit is contained in:
吉鑫升 committed 2026-08-27 00:09:53 +08:00
1 parent 4ce2f3a502
commit 485a8fa9d9
12 files changed
+410 -60

No files matched your search

+14 -1
View File
@@ -324,7 +324,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`. |
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering (`一、` / `(一)` / `1.`), two-character first-line indent on heading levels 1–3, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. Before numbering it removes complete manual heading prefixes (including `2.1` / `2.1.1`) and drops Markdown horizontal rules from Word output. The default ordinary-Q&A Lead Agent prompt emits the matching Markdown heading forms and avoids unordered bullet lists; writing setup, writing mode, notebook, scheduled runs, and SOUL-driven custom agents keep their independent prompts. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`, `tests/test_es_query_routing_prompt.py`. |
| **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. |
| **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/<name>` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. |
| **Suggestions** (`/api/threads/{id}/suggestions`) | `POST /` - generate follow-up questions; rich list/block model content is normalized before JSON parsing |
@@ -1611,3 +1611,16 @@ write, log the exception, complete the rewrite with
`versionSnapshotFailed: true`, and do not emit a user-facing failure or restore
the original. The sandbox uses that signal only to enable its ordinary manual
save fallback.
### Offline local-sandbox Office runtime
The DeerFlow-only offline deployment uses `LocalSandboxProvider`, so document
dependencies belong in the backend image rather than a separate sandbox image.
`deploy/linux-docker-offline/Dockerfile.backend` installs python-docx,
LibreOffice Writer/Calc/Impress, Pandoc, Poppler, and CJK fonts in addition to
the MarkItDown Office extras. `file_conversion.py` normalizes legacy
DOC/XLS/PPT and ODT/ODS/ODP through a per-conversion LibreOffice user profile
before MarkItDown extraction, avoiding concurrent profile locks. The image
build and `OFFLINE_INSTALL.sh --office-check` both run
`scripts/office_runtime_selfcheck.py`; that smoke test creates and re-parses
minimal DOCX/XLSX/PPTX files and does not need the application database.
@@ -60,7 +60,10 @@ RUN python -m venv /opt/venv \
&& /opt/venv/bin/python -m pip install --no-cache-dir --upgrade -i "${UV_INDEX_URL}" pip setuptools wheel \
&& sed -E '/--hash=/d; s/[[:space:]]+\\$//; s|^-e[[:space:]]+./packages/harness$|./packages/harness|' requirements-offline.txt > /tmp/requirements-docker.txt \
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" -r /tmp/requirements-docker.txt \
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" "asyncmy>=0.2.11"
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" \
"asyncmy>=0.2.11" \
"python-docx==1.2.0" \
&& /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd; print("Office Python dependencies OK")'
COPY . .
@@ -92,9 +95,17 @@ RUN if [ -n "${APT_MIRROR}" ]; then \
ca-certificates \
curl \
docker.io \
file \
fontconfig \
fonts-liberation \
fonts-noto-cjk \
libmagic1 \
libreoffice-calc \
libreoffice-impress \
libreoffice-writer \
nodejs \
npm \
pandoc \
poppler-utils \
tini \
&& rm -rf /var/lib/apt/lists/*
@@ -102,6 +113,11 @@ RUN if [ -n "${APT_MIRROR}" ]; then \
COPY --from=builder /opt/venv /opt/venv
COPY --from=builder /build/backend /app/backend
RUN /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd' \
&& command -v soffice >/dev/null \
&& command -v pandoc >/dev/null \
&& fc-list :lang=zh | grep -q .
EXPOSE 8001
ENTRYPOINT ["tini", "--"]
+11 -1
View File
@@ -156,7 +156,7 @@ FastAPI application providing REST endpoints for frontend integration:
| `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. |
| `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. |
| `POST /api/ai-writing/sample/extract` | Extract text from Word/PDF/Markdown/TXT samples for AI-writing imitation mode |
| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. |
| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, removes manual/compound heading prefixes and Markdown horizontal rules, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. |
| `POST /cop/saveSuperiorTask` | TaskCOP compatibility API (DeerFlow Bearer/session authentication required): create a situation-overview task with a sequential four-digit id (`0001`, `0002`, ...) and return the legacy `{state, msg, data}` envelope |
| `POST /api/taskcop/import/tasks` | TaskCOP compatibility API (authenticated): import tasks from a JSON array or `tasks`/`records` envelope; every row's supplied `id`/`taskId` is authoritative and a duplicate id completely replaces the stored task |
| `DELETE /cop/tasks/{task_id}` | TaskCOP compatibility API (authenticated): delete one task and its task-scoped situation-report document |
@@ -943,3 +943,13 @@ The final rewrite commit uses a deliberately short sibling temporary filename,
which keeps long Windows artifact paths below the path-length limit. It also
recreates a missing `outputs` parent before retrying, so transient
sandbox-directory recreation cannot discard a completed rewrite.
## Offline local-sandbox Office runtime
The pure DeerFlow offline image treats the backend container as the default
`LocalSandboxProvider` runtime. It bundles MarkItDown, python-docx, openpyxl,
python-pptx, LibreOffice Headless, Pandoc, Poppler, and CJK fonts. Modern OOXML
uploads are parsed directly; legacy DOC/XLS/PPT and ODT/ODS/ODP files are
normalized through an isolated LibreOffice profile before Markdown extraction.
Run `./OFFLINE_INSTALL.sh --office-check` from the offline bundle to generate
and re-parse Word, Excel, and PowerPoint samples without requiring a database.
@@ -214,7 +214,13 @@ def _strip_inline_markdown(text: str) -> str:
def _strip_manual_heading_number(text: str) -> str:
return re.sub(
r"^\s*(?:[一二三四五六七八九十百千]+、|[((][一二三四五六七八九十百千]+[))]|\d+[.、]|[((]\d+[))]|\d+[))])\s*",
r"^\s*(?:"
r"[一二三四五六七八九十百千]+[、..]"
r"|[((]\s*[一二三四五六七八九十百千]+\s*[))]"
r"|[((]\s*\d+(?:[..]\d+)*\s*[))]"
r"|\d+(?:[..]\d+)+(?:[..、])?"
r"|\d+[..、))]"
r")\s*",
"",
text,
).strip()
@@ -338,7 +344,6 @@ def _parse_markdown(markdown: str, explicit_title: str | None) -> tuple[str | No
if re.fullmatch(r"\s*(?:---|\*\*\*|___)\s*", raw):
flush_paragraph()
blocks.append(ParagraphBlock("horizontal_rule", "────────"))
index += 1
continue
@@ -584,8 +589,6 @@ def _blocks_xml(blocks: list[Block], relationships: _DocumentRelationships) -> s
output.append(_paragraph_xml("QuoteText", _inline_xml(block.text, relationships)))
elif block.kind == "image":
output.append(_image_paragraph_xml(block.text, relationships))
elif block.kind == "horizontal_rule":
output.append(_paragraph_xml("BodyText", _run_xml(block.text), alignment="center"))
else:
output.append(_paragraph_xml("BodyText", _inline_xml(block.text, relationships)))
return "".join(output)
@@ -677,6 +680,7 @@ def _styles_xml() -> str:
font=HEADING_1_FONT_FAMILY,
size=BODY_SIZE_HALF_POINTS,
alignment="left",
first_line_chars=200,
keep_next=True,
keep_lines=True,
outline_level=0,
@@ -687,6 +691,7 @@ def _styles_xml() -> str:
font=HEADING_2_FONT_FAMILY,
size=BODY_SIZE_HALF_POINTS,
alignment="left",
first_line_chars=200,
keep_next=True,
keep_lines=True,
outline_level=1,
@@ -700,6 +705,7 @@ def _styles_xml() -> str:
font=BODY_FONT_FAMILY,
size=BODY_SIZE_HALF_POINTS,
alignment="left",
first_line_chars=200 if level == 3 else 0,
keep_next=True,
keep_lines=True,
outline_level=level - 1,
@@ -769,10 +775,11 @@ def _styles_xml() -> str:
def _heading_level_xml(level: int, format_value: str, text: str, font: str) -> str:
restart = f'<w:lvlRestart w:val="{level}"/>' if level > 0 else ""
first_line_chars = ' w:firstLineChars="200"' if level <= 2 else ""
return (
f'<w:lvl w:ilvl="{level}"><w:start w:val="1"/>{restart}<w:numFmt w:val="{format_value}"/>'
f'<w:lvlText w:val="{_xml_attr(text)}"/><w:lvlJc w:val="left"/>'
'<w:pPr><w:ind w:left="0" w:hanging="0"/></w:pPr>'
f'<w:pPr><w:ind w:left="0" w:hanging="0"{first_line_chars}/></w:pPr>'
f'<w:rPr>{_font_run_properties(font, BODY_SIZE_HALF_POINTS)}</w:rPr></w:lvl>'
)
@@ -798,7 +805,7 @@ def _numbering_xml() -> str:
heading_levels = "".join(
[
_heading_level_xml(0, "chineseCounting", "%1、", HEADING_1_FONT_FAMILY),
_heading_level_xml(1, "chineseCounting", "(%2)", HEADING_2_FONT_FAMILY),
_heading_level_xml(1, "chineseCounting", "(%2)", HEADING_2_FONT_FAMILY),
_heading_level_xml(2, "decimal", "%3.", BODY_FONT_FAMILY),
_heading_level_xml(3, "decimal", "(%4)", BODY_FONT_FAMILY),
_heading_level_xml(4, "decimal", "%5)", BODY_FONT_FAMILY),
@@ -106,13 +106,14 @@ DELETE /api/threads/{thread_id}/uploads/{filename}
以下格式在显式启用 `uploads.auto_convert_documents: true` 时会自动转换为 Markdown:
- PDF (`.pdf`)
- PowerPoint (`.ppt`, `.pptx`)
- Excel (`.xls`, `.xlsx`)
- Word (`.doc`, `.docx`)
- PowerPoint (`.ppt`, `.pptx`)
- Excel (`.xls`, `.xlsx`)
- Word (`.doc`, `.docx`)
- OpenDocument (`.odt`, `.ods`, `.odp`)
转换后的 Markdown 文件会保存在同一目录下,文件名为原文件名 + `.md` 扩展名。
默认情况下,自动转换是关闭的,以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。只有在受信任部署中明确接受此风险时,才应将 `uploads.auto_convert_documents` 设置为 `true`。
通用部署可以关闭自动转换,以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。本仓库当前 `config.yaml` 和纯 DeerFlow 离线部署包默认设置为 `uploads.auto_convert_documents: true`,用于受信任的内网部署。
## Agent 集成
@@ -247,8 +248,15 @@ backend/.deer-flow/threads/
### 依赖
- `markitdown>=0.0.1a2` - 文档转换
- `python-multipart>=0.0.20` - 文件上传处理
- `markitdown>=0.0.1a2` - 文档转换
- `python-multipart>=0.0.20` - 文件上传处理
- 纯 DeerFlow 离线镜像还内置 `python-docx`、`openpyxl`、`python-pptx`、LibreOffice Headless、Pandoc、Poppler 和中文字体。旧版 DOC/XLS/PPT 及 ODT/ODS/ODP 会先经 LibreOffice 规范化,再由 MarkItDown 提取。
离线部署包可在不连接数据库的情况下运行依赖自检:
```bash
./OFFLINE_INSTALL.sh --office-check
```
## 故障排查
@@ -686,10 +686,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
available_skills=prompt_available_skills,
app_config=resolved_app_config,
memory_injection_enabled=memory_injection_enabled,
rag_mode_enabled=rag_mode_enabled,
is_notebook_mode=is_notebook_mode,
notebook_source_ids=notebook_source_ids,
)
rag_mode_enabled=rag_mode_enabled,
is_notebook_mode=is_notebook_mode,
notebook_source_ids=notebook_source_ids,
is_scheduled_run=bool(cfg.get("is_scheduled_run", False)),
)
return create_agent(
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, force_disable_thinking=thinking_force_disabled, app_config=resolved_app_config),
@@ -486,7 +486,19 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p
</subagent_system>"""
SYSTEM_PROMPT_TEMPLATE = """
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION = """
<ordinary_qa_markdown_format>
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。二级编号必须使用半角括号 `(一)`。
- 不要在标题中使用 `2.1`、`2.1.1` 等多级阿拉伯数字编号;章节编号直接按上述三种形式书写。
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
</ordinary_qa_markdown_format>
"""
SYSTEM_PROMPT_TEMPLATE = """
<role>
You are {agent_name}, an internal super agent.
</role>
@@ -612,13 +624,15 @@ You: "Deploying to staging..." [proceed]
{acp_section}
</working_directory>
<response_style>
- Clear and Concise: Avoid over-formatting unless requested
- Natural Tone: Use paragraphs and prose, not bullet points by default
- Action-Oriented: Focus on delivering results, not explaining processes
</response_style>
<citations>
<response_style>
- Clear and Concise: Avoid over-formatting unless requested
- Natural Tone: Use paragraphs and prose, not bullet points by default
- Action-Oriented: Focus on delivering results, not explaining processes
</response_style>
{ordinary_qa_format_section}
<citations>
**CRITICAL: Always include citations when using web search results**
- **When to Use**: MANDATORY after web_search, web_fetch, or any external information source
@@ -1374,7 +1388,7 @@ def build_report_structure_setup_prompt(
"""
def apply_prompt_template(
def apply_prompt_template(
subagent_enabled: bool = False,
max_concurrent_subagents: int = 3,
*,
@@ -1386,10 +1400,11 @@ def apply_prompt_template(
is_bootstrap: bool = False,
app_config: AppConfig | None = None,
memory_injection_enabled: bool = True,
rag_mode_enabled: bool = False,
is_notebook_mode: bool = False,
notebook_source_ids: list[int] | None = None,
) -> str:
rag_mode_enabled: bool = False,
is_notebook_mode: bool = False,
notebook_source_ids: list[int] | None = None,
is_scheduled_run: bool = False,
) -> str:
storage_agent_id = agent_id if agent_id is not None else agent_name
# Get memory context
@@ -1467,7 +1482,7 @@ Writing mode is enabled for this run.
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
network_policy_section = _build_network_policy_section()
rag_mode_section = (
rag_mode_section = (
"""
<rag_citation_mode>
RAG/reference display mode is enabled.
@@ -1480,10 +1495,15 @@ When a search, knowledge, skill, or MCP tool returns a result list:
</rag_citation_mode>
"""
if rag_mode_enabled
else ""
)
# Format the prompt with dynamic skills and memory
else ""
)
ordinary_qa_format_section = (
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION
if not is_bootstrap and not writing_mode and not is_notebook_mode and not is_scheduled_run
else ""
)
# Format the prompt with dynamic skills and memory
prompt = SYSTEM_PROMPT_TEMPLATE.format(
agent_name=agent_name or storage_agent_id or "CM 助手",
soul=get_agent_soul(storage_agent_id),
@@ -1496,9 +1516,10 @@ When a search, knowledge, skill, or MCP tool returns a result list:
network_policy_section=network_policy_section,
subagent_section=subagent_section,
subagent_reminder=subagent_reminder,
subagent_thinking=subagent_thinking,
acp_section=acp_and_mounts_section,
)
subagent_thinking=subagent_thinking,
acp_section=acp_and_mounts_section,
ordinary_qa_format_section=ordinary_qa_format_section,
)
prompt = prompt + rag_mode_section + _current_time_context()
@@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
No FastAPI or HTTP dependencies — pure utility functions.
"""
import asyncio
import logging
import re
from pathlib import Path
import asyncio
import logging
import os
import re
import shutil
import subprocess
import tempfile
from pathlib import Path
from deerflow.config.app_config import get_app_config
logger = logging.getLogger(__name__)
# File extensions that should be converted to markdown
CONVERTIBLE_EXTENSIONS = {
CONVERTIBLE_EXTENSIONS = {
".pdf",
".ppt",
".pptx",
".xls",
".xlsx",
".doc",
".docx",
}
".docx",
".odt",
".ods",
".odp",
}
# MarkItDown reads modern OOXML files directly. LibreOffice is used as a
# compatibility bridge for legacy binary Office files and OpenDocument files,
# which are common in offline/intranet deployments but are not handled
# consistently by the pure-Python readers.
_LIBREOFFICE_TARGETS = {
".doc": ".docx",
".ppt": ".pptx",
".xls": ".xlsx",
".odt": ".docx",
".odp": ".pptx",
".ods": ".xlsx",
}
_LIBREOFFICE_TIMEOUT_SECONDS = 120
# Files larger than this threshold are converted in a background thread.
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
@@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None:
return None
def _convert_with_markitdown(file_path: Path) -> str:
def _convert_with_markitdown(file_path: Path) -> str:
"""Convert any supported file to markdown text using MarkItDown."""
from markitdown import MarkItDown
md = MarkItDown()
return md.convert(str(file_path)).text_content
def _do_convert(file_path: Path, pdf_converter: str) -> str:
return md.convert(str(file_path)).text_content
def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
"""Normalize a legacy Office/OpenDocument file, then extract its text.
Each conversion gets an isolated LibreOffice user profile. This avoids
the profile-lock conflict that otherwise appears when several uploads are
converted concurrently in the same sandbox container.
"""
target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower())
if target_suffix is None:
return None
executable = shutil.which("soffice") or shutil.which("libreoffice")
if executable is None:
logger.warning(
"LibreOffice is unavailable; trying MarkItDown directly for %s",
file_path.name,
)
return None
with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary:
temp_root = Path(temporary).resolve()
output_dir = temp_root / "output"
profile_dir = temp_root / "profile"
output_dir.mkdir()
profile_dir.mkdir()
command = [
executable,
"--headless",
"--nologo",
"--nodefault",
"--nolockcheck",
"--nofirststartwizard",
f"-env:UserInstallation={profile_dir.as_uri()}",
"--convert-to",
target_suffix.lstrip("."),
"--outdir",
str(output_dir),
str(file_path.resolve()),
]
try:
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
timeout=_LIBREOFFICE_TIMEOUT_SECONDS,
env={**os.environ, "HOME": str(temp_root)},
)
except (OSError, subprocess.TimeoutExpired):
logger.exception("LibreOffice failed to normalize %s", file_path.name)
return None
converted_path = output_dir / f"{file_path.stem}{target_suffix}"
if completed.returncode != 0 or not converted_path.is_file():
logger.error(
"LibreOffice could not normalize %s (exit=%s, stderr=%s)",
file_path.name,
completed.returncode,
completed.stderr.strip()[-1000:],
)
return None
return _convert_with_markitdown(converted_path)
def _do_convert(file_path: Path, pdf_converter: str) -> str:
"""Synchronous conversion — called directly or via asyncio.to_thread.
Args:
file_path: Path to the file.
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
"""
is_pdf = file_path.suffix.lower() == ".pdf"
if is_pdf and pdf_converter != "markitdown":
is_pdf = file_path.suffix.lower() == ".pdf"
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
if normalized_office_text is not None:
return normalized_office_text
if is_pdf and pdf_converter != "markitdown":
# Try pymupdf4llm first (auto or explicit)
pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path)
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Offline runtime smoke test for the LocalSandbox Office toolchain."""
from __future__ import annotations
import shutil
import subprocess
import tempfile
from pathlib import Path
def require_command(name: str) -> str:
path = shutil.which(name)
if path is None:
raise RuntimeError(f"missing command: {name}")
return path
def assert_contains(text: str, expected: str, label: str) -> None:
normalized = text.replace("\\_", "_")
if expected not in normalized:
raise RuntimeError(f"{label} conversion lost test text: {text[:300]!r}")
def main() -> int:
from docx import Document
from markitdown import MarkItDown
from openpyxl import Workbook
from pptx import Presentation
soffice = require_command("soffice")
require_command("pandoc")
require_command("pdftotext")
require_command("file")
subprocess.run(
[soffice, "--headless", "--version"],
check=True,
capture_output=True,
text=True,
timeout=30,
)
with tempfile.TemporaryDirectory(prefix="deerflow-office-selfcheck-") as temporary:
root = Path(temporary)
word_path = root / "word-test.docx"
document = Document()
document.add_heading("Word 自检", level=1)
document.add_paragraph("DEERFLOW_WORD_OK")
document.save(word_path)
excel_path = root / "excel-test.xlsx"
workbook = Workbook()
worksheet = workbook.active
worksheet.title = "自检"
worksheet.append(["状态", "DEERFLOW_EXCEL_OK"])
workbook.save(excel_path)
ppt_path = root / "ppt-test.pptx"
presentation = Presentation()
slide = presentation.slides.add_slide(presentation.slide_layouts[5])
slide.shapes.title.text = "DEERFLOW_PPT_OK"
presentation.save(ppt_path)
converter = MarkItDown()
assert_contains(converter.convert(word_path).text_content, "DEERFLOW_WORD_OK", "Word")
assert_contains(converter.convert(excel_path).text_content, "DEERFLOW_EXCEL_OK", "Excel")
assert_contains(converter.convert(ppt_path).text_content, "DEERFLOW_PPT_OK", "PowerPoint")
print("OFFICE_RUNTIME_SELF_CHECK=PASS")
print("支持:DOC/DOCX、XLS/XLSX、PPT/PPTX、ODT/ODS/ODP、PDF")
print("组件:MarkItDown、python-docx、openpyxl、python-pptx、LibreOffice、Pandoc、中文字体")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -68,3 +68,29 @@ def test_es_query_routing_is_injected_into_lead_and_allowed_custom_agent_prompts
assert "自定义 ES 路由规则" in lead_prompt
assert '<skill_routing skill="es_query">' in allowed_custom_prompt
assert '<skill_routing skill="es_query">' not in denied_custom_prompt
def test_formal_markdown_rules_are_scoped_to_ordinary_qa(monkeypatch) -> None:
config = _app_config(enabled=False)
monkeypatch.setattr(prompt_module, "_get_memory_context", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "get_agent_soul", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "get_skills_prompt_section", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "_build_acp_section", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "_build_custom_mounts_section", lambda *args, **kwargs: "")
monkeypatch.setattr(prompt_module, "_current_time_context", lambda: "")
ordinary_prompt = prompt_module.apply_prompt_template(app_config=config)
writing_prompt = prompt_module.apply_prompt_template(app_config=config, writing_mode=True)
notebook_prompt = prompt_module.apply_prompt_template(app_config=config, is_notebook_mode=True)
scheduled_prompt = prompt_module.apply_prompt_template(app_config=config, is_scheduled_run=True)
assert '<ordinary_qa_markdown_format>' in ordinary_prompt
assert "## 一、标题" in ordinary_prompt
assert "### (一) 标题" in ordinary_prompt
assert "#### 1. 标题" in ordinary_prompt
assert "2.1.1" in ordinary_prompt
assert "无序圆点列表" in ordinary_prompt
assert '<ordinary_qa_markdown_format>' not in writing_prompt
assert '<ordinary_qa_markdown_format>' not in notebook_prompt
assert '<ordinary_qa_markdown_format>' not in scheduled_prompt
@@ -0,0 +1,57 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from deerflow.utils import file_conversion
def test_legacy_word_is_normalized_before_markitdown(monkeypatch, tmp_path: Path) -> None:
source = tmp_path / "legacy.doc"
source.write_bytes(b"legacy")
monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: "/usr/bin/soffice")
monkeypatch.setattr(
file_conversion,
"_convert_with_markitdown",
lambda path: f"converted:{path.suffix}",
)
def fake_run(command, **_kwargs):
output_dir = Path(command[command.index("--outdir") + 1])
(output_dir / "legacy.docx").write_bytes(b"normalized")
return SimpleNamespace(returncode=0, stderr="")
monkeypatch.setattr(file_conversion.subprocess, "run", fake_run)
assert file_conversion._do_convert(source, "auto") == "converted:.docx"
def test_modern_office_file_does_not_start_libreoffice(monkeypatch, tmp_path: Path) -> None:
source = tmp_path / "modern.pptx"
source.write_bytes(b"modern")
monkeypatch.setattr(
file_conversion,
"_convert_with_markitdown",
lambda path: f"direct:{path.suffix}",
)
monkeypatch.setattr(
file_conversion.subprocess,
"run",
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected LibreOffice call")),
)
assert file_conversion._do_convert(source, "auto") == "direct:.pptx"
def test_legacy_office_falls_back_when_libreoffice_is_missing(monkeypatch, tmp_path: Path) -> None:
source = tmp_path / "legacy.xls"
source.write_bytes(b"legacy")
monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: None)
monkeypatch.setattr(
file_conversion,
"_convert_with_markitdown",
lambda path: f"fallback:{path.suffix}",
)
assert file_conversion._do_convert(source, "auto") == "fallback:.xls"
@@ -41,8 +41,10 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
## 一、一级标题
这是正文,用于验证首行缩进、两端对齐和固定行距。
### (一)二级标题
#### 1. 三级标题
### 2.1 二级标题
#### 2.1.1 三级标题
---
表1 测试表格
| 项目 | 数值 |
@@ -77,8 +79,11 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
assert '<w:pStyle w:val="DocumentTitle"/>' in document
assert "方正小标宋简体" in document
assert "一、一级标题" not in document
assert "(一)二级标题" not in document
assert "1. 三级标题" not in document
assert "2.1 二级标题" not in document
assert "2.1.1 三级标题" not in document
assert "────────" not in document
assert "二级标题" in document
assert "三级标题" in document
assert 'w:line="579" w:lineRule="exact"' in styles
assert '<w:ind w:left="0" w:hanging="0" w:firstLineChars="200"/>' in styles
@@ -86,11 +91,13 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
assert 'w:val="44"' in styles
assert "仿宋" in styles and 'w:val="32"' in styles
assert "宋体" in styles and 'w:val="28"' in styles
assert styles.count('w:firstLineChars="200"') >= 6
assert '<w:numFmt w:val="chineseCounting"/>' in numbering
assert '<w:lvlText w:val="%1、"/>' in numbering
assert '<w:lvlText w:val="(%2)"/>' in numbering
assert '<w:lvlText w:val="(%2)"/>' in numbering
assert '<w:lvlText w:val="%3."/>' in numbering
assert numbering.count('w:firstLineChars="200"') == 3
assert '<w:evenAndOddHeaders/>' in settings
assert '<w:updateFields w:val="true"/>' in settings
@@ -114,3 +121,20 @@ def test_export_rejects_wrong_font(tmp_path: Path) -> None:
with pytest.raises(word_export.WordExportFontError):
word_export.build_markdown_docx("正文", font_path=wrong_font)
@pytest.mark.parametrize(
("source", "expected"),
[
("一、一级标题", "一级标题"),
("(一) 二级标题", "二级标题"),
("(一)二级标题", "二级标题"),
("1. 三级标题", "三级标题"),
("2.1 二级标题", "二级标题"),
("2.1.1 三级标题", "三级标题"),
("2.1.1. 三级标题", "三级标题"),
("正文标题", "正文标题"),
],
)
def test_manual_heading_number_is_removed_as_one_complete_prefix(source: str, expected: str) -> None:
assert word_export._strip_manual_heading_number(source) == expected