feat: standardize Word export and Office conversion
This commit is contained in:
1 parent
4ce2f3a502
commit
485a8fa9d9
12 files changed
+410
-60
No files matched your search
@@ -324,7 +324,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
|
||||
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
|
||||
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
|
||||
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
|
||||
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`. |
|
||||
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering (`一、` / `(一)` / `1.`), two-character first-line indent on heading levels 1–3, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. Before numbering it removes complete manual heading prefixes (including `2.1` / `2.1.1`) and drops Markdown horizontal rules from Word output. The default ordinary-Q&A Lead Agent prompt emits the matching Markdown heading forms and avoids unordered bullet lists; writing setup, writing mode, notebook, scheduled runs, and SOUL-driven custom agents keep their independent prompts. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`, `tests/test_es_query_routing_prompt.py`. |
|
||||
| **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. |
|
||||
| **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/<name>` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. |
|
||||
| **Suggestions** (`/api/threads/{id}/suggestions`) | `POST /` - generate follow-up questions; rich list/block model content is normalized before JSON parsing |
|
||||
@@ -1611,3 +1611,16 @@ write, log the exception, complete the rewrite with
|
||||
`versionSnapshotFailed: true`, and do not emit a user-facing failure or restore
|
||||
the original. The sandbox uses that signal only to enable its ordinary manual
|
||||
save fallback.
|
||||
|
||||
### Offline local-sandbox Office runtime
|
||||
|
||||
The DeerFlow-only offline deployment uses `LocalSandboxProvider`, so document
|
||||
dependencies belong in the backend image rather than a separate sandbox image.
|
||||
`deploy/linux-docker-offline/Dockerfile.backend` installs python-docx,
|
||||
LibreOffice Writer/Calc/Impress, Pandoc, Poppler, and CJK fonts in addition to
|
||||
the MarkItDown Office extras. `file_conversion.py` normalizes legacy
|
||||
DOC/XLS/PPT and ODT/ODS/ODP through a per-conversion LibreOffice user profile
|
||||
before MarkItDown extraction, avoiding concurrent profile locks. The image
|
||||
build and `OFFLINE_INSTALL.sh --office-check` both run
|
||||
`scripts/office_runtime_selfcheck.py`; that smoke test creates and re-parses
|
||||
minimal DOCX/XLSX/PPTX files and does not need the application database.
|
||||
@@ -60,7 +60,10 @@ RUN python -m venv /opt/venv \
|
||||
&& /opt/venv/bin/python -m pip install --no-cache-dir --upgrade -i "${UV_INDEX_URL}" pip setuptools wheel \
|
||||
&& sed -E '/--hash=/d; s/[[:space:]]+\\$//; s|^-e[[:space:]]+./packages/harness$|./packages/harness|' requirements-offline.txt > /tmp/requirements-docker.txt \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" -r /tmp/requirements-docker.txt \
|
||||
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" "asyncmy>=0.2.11"
|
||||
&& /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" \
|
||||
"asyncmy>=0.2.11" \
|
||||
"python-docx==1.2.0" \
|
||||
&& /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd; print("Office Python dependencies OK")'
|
||||
|
||||
COPY . .
|
||||
|
||||
@@ -92,9 +95,17 @@ RUN if [ -n "${APT_MIRROR}" ]; then \
|
||||
ca-certificates \
|
||||
curl \
|
||||
docker.io \
|
||||
file \
|
||||
fontconfig \
|
||||
fonts-liberation \
|
||||
fonts-noto-cjk \
|
||||
libmagic1 \
|
||||
libreoffice-calc \
|
||||
libreoffice-impress \
|
||||
libreoffice-writer \
|
||||
nodejs \
|
||||
npm \
|
||||
pandoc \
|
||||
poppler-utils \
|
||||
tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
@@ -102,6 +113,11 @@ RUN if [ -n "${APT_MIRROR}" ]; then \
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
COPY --from=builder /build/backend /app/backend
|
||||
|
||||
RUN /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd' \
|
||||
&& command -v soffice >/dev/null \
|
||||
&& command -v pandoc >/dev/null \
|
||||
&& fc-list :lang=zh | grep -q .
|
||||
|
||||
EXPOSE 8001
|
||||
|
||||
ENTRYPOINT ["tini", "--"]
|
||||
|
||||
@@ -156,7 +156,7 @@ FastAPI application providing REST endpoints for frontend integration:
|
||||
| `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. |
|
||||
| `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. |
|
||||
| `POST /api/ai-writing/sample/extract` | Extract text from Word/PDF/Markdown/TXT samples for AI-writing imitation mode |
|
||||
| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. |
|
||||
| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, removes manual/compound heading prefixes and Markdown horizontal rules, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. |
|
||||
| `POST /cop/saveSuperiorTask` | TaskCOP compatibility API (DeerFlow Bearer/session authentication required): create a situation-overview task with a sequential four-digit id (`0001`, `0002`, ...) and return the legacy `{state, msg, data}` envelope |
|
||||
| `POST /api/taskcop/import/tasks` | TaskCOP compatibility API (authenticated): import tasks from a JSON array or `tasks`/`records` envelope; every row's supplied `id`/`taskId` is authoritative and a duplicate id completely replaces the stored task |
|
||||
| `DELETE /cop/tasks/{task_id}` | TaskCOP compatibility API (authenticated): delete one task and its task-scoped situation-report document |
|
||||
@@ -943,3 +943,13 @@ The final rewrite commit uses a deliberately short sibling temporary filename,
|
||||
which keeps long Windows artifact paths below the path-length limit. It also
|
||||
recreates a missing `outputs` parent before retrying, so transient
|
||||
sandbox-directory recreation cannot discard a completed rewrite.
|
||||
|
||||
## Offline local-sandbox Office runtime
|
||||
|
||||
The pure DeerFlow offline image treats the backend container as the default
|
||||
`LocalSandboxProvider` runtime. It bundles MarkItDown, python-docx, openpyxl,
|
||||
python-pptx, LibreOffice Headless, Pandoc, Poppler, and CJK fonts. Modern OOXML
|
||||
uploads are parsed directly; legacy DOC/XLS/PPT and ODT/ODS/ODP files are
|
||||
normalized through an isolated LibreOffice profile before Markdown extraction.
|
||||
Run `./OFFLINE_INSTALL.sh --office-check` from the offline bundle to generate
|
||||
and re-parse Word, Excel, and PowerPoint samples without requiring a database.
|
||||
@@ -214,7 +214,13 @@ def _strip_inline_markdown(text: str) -> str:
|
||||
|
||||
def _strip_manual_heading_number(text: str) -> str:
|
||||
return re.sub(
|
||||
r"^\s*(?:[一二三四五六七八九十百千]+、|[((][一二三四五六七八九十百千]+[))]|\d+[.、]|[((]\d+[))]|\d+[))])\s*",
|
||||
r"^\s*(?:"
|
||||
r"[一二三四五六七八九十百千]+[、..]"
|
||||
r"|[((]\s*[一二三四五六七八九十百千]+\s*[))]"
|
||||
r"|[((]\s*\d+(?:[..]\d+)*\s*[))]"
|
||||
r"|\d+(?:[..]\d+)+(?:[..、])?"
|
||||
r"|\d+[..、))]"
|
||||
r")\s*",
|
||||
"",
|
||||
text,
|
||||
).strip()
|
||||
@@ -338,7 +344,6 @@ def _parse_markdown(markdown: str, explicit_title: str | None) -> tuple[str | No
|
||||
|
||||
if re.fullmatch(r"\s*(?:---|\*\*\*|___)\s*", raw):
|
||||
flush_paragraph()
|
||||
blocks.append(ParagraphBlock("horizontal_rule", "────────"))
|
||||
index += 1
|
||||
continue
|
||||
|
||||
@@ -584,8 +589,6 @@ def _blocks_xml(blocks: list[Block], relationships: _DocumentRelationships) -> s
|
||||
output.append(_paragraph_xml("QuoteText", _inline_xml(block.text, relationships)))
|
||||
elif block.kind == "image":
|
||||
output.append(_image_paragraph_xml(block.text, relationships))
|
||||
elif block.kind == "horizontal_rule":
|
||||
output.append(_paragraph_xml("BodyText", _run_xml(block.text), alignment="center"))
|
||||
else:
|
||||
output.append(_paragraph_xml("BodyText", _inline_xml(block.text, relationships)))
|
||||
return "".join(output)
|
||||
@@ -677,6 +680,7 @@ def _styles_xml() -> str:
|
||||
font=HEADING_1_FONT_FAMILY,
|
||||
size=BODY_SIZE_HALF_POINTS,
|
||||
alignment="left",
|
||||
first_line_chars=200,
|
||||
keep_next=True,
|
||||
keep_lines=True,
|
||||
outline_level=0,
|
||||
@@ -687,6 +691,7 @@ def _styles_xml() -> str:
|
||||
font=HEADING_2_FONT_FAMILY,
|
||||
size=BODY_SIZE_HALF_POINTS,
|
||||
alignment="left",
|
||||
first_line_chars=200,
|
||||
keep_next=True,
|
||||
keep_lines=True,
|
||||
outline_level=1,
|
||||
@@ -700,6 +705,7 @@ def _styles_xml() -> str:
|
||||
font=BODY_FONT_FAMILY,
|
||||
size=BODY_SIZE_HALF_POINTS,
|
||||
alignment="left",
|
||||
first_line_chars=200 if level == 3 else 0,
|
||||
keep_next=True,
|
||||
keep_lines=True,
|
||||
outline_level=level - 1,
|
||||
@@ -769,10 +775,11 @@ def _styles_xml() -> str:
|
||||
|
||||
def _heading_level_xml(level: int, format_value: str, text: str, font: str) -> str:
|
||||
restart = f'<w:lvlRestart w:val="{level}"/>' if level > 0 else ""
|
||||
first_line_chars = ' w:firstLineChars="200"' if level <= 2 else ""
|
||||
return (
|
||||
f'<w:lvl w:ilvl="{level}"><w:start w:val="1"/>{restart}<w:numFmt w:val="{format_value}"/>'
|
||||
f'<w:lvlText w:val="{_xml_attr(text)}"/><w:lvlJc w:val="left"/>'
|
||||
'<w:pPr><w:ind w:left="0" w:hanging="0"/></w:pPr>'
|
||||
f'<w:pPr><w:ind w:left="0" w:hanging="0"{first_line_chars}/></w:pPr>'
|
||||
f'<w:rPr>{_font_run_properties(font, BODY_SIZE_HALF_POINTS)}</w:rPr></w:lvl>'
|
||||
)
|
||||
|
||||
@@ -798,7 +805,7 @@ def _numbering_xml() -> str:
|
||||
heading_levels = "".join(
|
||||
[
|
||||
_heading_level_xml(0, "chineseCounting", "%1、", HEADING_1_FONT_FAMILY),
|
||||
_heading_level_xml(1, "chineseCounting", "(%2)", HEADING_2_FONT_FAMILY),
|
||||
_heading_level_xml(1, "chineseCounting", "(%2)", HEADING_2_FONT_FAMILY),
|
||||
_heading_level_xml(2, "decimal", "%3.", BODY_FONT_FAMILY),
|
||||
_heading_level_xml(3, "decimal", "(%4)", BODY_FONT_FAMILY),
|
||||
_heading_level_xml(4, "decimal", "%5)", BODY_FONT_FAMILY),
|
||||
|
||||
@@ -106,13 +106,14 @@ DELETE /api/threads/{thread_id}/uploads/{filename}
|
||||
|
||||
以下格式在显式启用 `uploads.auto_convert_documents: true` 时会自动转换为 Markdown:
|
||||
- PDF (`.pdf`)
|
||||
- PowerPoint (`.ppt`, `.pptx`)
|
||||
- Excel (`.xls`, `.xlsx`)
|
||||
- Word (`.doc`, `.docx`)
|
||||
- PowerPoint (`.ppt`, `.pptx`)
|
||||
- Excel (`.xls`, `.xlsx`)
|
||||
- Word (`.doc`, `.docx`)
|
||||
- OpenDocument (`.odt`, `.ods`, `.odp`)
|
||||
|
||||
转换后的 Markdown 文件会保存在同一目录下,文件名为原文件名 + `.md` 扩展名。
|
||||
|
||||
默认情况下,自动转换是关闭的,以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。只有在受信任部署中明确接受此风险时,才应将 `uploads.auto_convert_documents` 设置为 `true`。
|
||||
通用部署可以关闭自动转换,以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。本仓库当前 `config.yaml` 和纯 DeerFlow 离线部署包默认设置为 `uploads.auto_convert_documents: true`,用于受信任的内网部署。
|
||||
|
||||
## Agent 集成
|
||||
|
||||
@@ -247,8 +248,15 @@ backend/.deer-flow/threads/
|
||||
|
||||
### 依赖
|
||||
|
||||
- `markitdown>=0.0.1a2` - 文档转换
|
||||
- `python-multipart>=0.0.20` - 文件上传处理
|
||||
- `markitdown>=0.0.1a2` - 文档转换
|
||||
- `python-multipart>=0.0.20` - 文件上传处理
|
||||
- 纯 DeerFlow 离线镜像还内置 `python-docx`、`openpyxl`、`python-pptx`、LibreOffice Headless、Pandoc、Poppler 和中文字体。旧版 DOC/XLS/PPT 及 ODT/ODS/ODP 会先经 LibreOffice 规范化,再由 MarkItDown 提取。
|
||||
|
||||
离线部署包可在不连接数据库的情况下运行依赖自检:
|
||||
|
||||
```bash
|
||||
./OFFLINE_INSTALL.sh --office-check
|
||||
```
|
||||
|
||||
## 故障排查
|
||||
|
||||
|
||||
+5
-4
@@ -686,10 +686,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
available_skills=prompt_available_skills,
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
)
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
is_scheduled_run=bool(cfg.get("is_scheduled_run", False)),
|
||||
)
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, force_disable_thinking=thinking_force_disabled, app_config=resolved_app_config),
|
||||
|
||||
+42
-21
@@ -486,7 +486,19 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p
|
||||
</subagent_system>"""
|
||||
|
||||
|
||||
SYSTEM_PROMPT_TEMPLATE = """
|
||||
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION = """
|
||||
<ordinary_qa_markdown_format>
|
||||
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
|
||||
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
|
||||
- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。二级编号必须使用半角括号 `(一)`。
|
||||
- 不要在标题中使用 `2.1`、`2.1.1` 等多级阿拉伯数字编号;章节编号直接按上述三种形式书写。
|
||||
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
|
||||
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
|
||||
</ordinary_qa_markdown_format>
|
||||
"""
|
||||
|
||||
|
||||
SYSTEM_PROMPT_TEMPLATE = """
|
||||
<role>
|
||||
You are {agent_name}, an internal super agent.
|
||||
</role>
|
||||
@@ -612,13 +624,15 @@ You: "Deploying to staging..." [proceed]
|
||||
{acp_section}
|
||||
</working_directory>
|
||||
|
||||
<response_style>
|
||||
- Clear and Concise: Avoid over-formatting unless requested
|
||||
- Natural Tone: Use paragraphs and prose, not bullet points by default
|
||||
- Action-Oriented: Focus on delivering results, not explaining processes
|
||||
</response_style>
|
||||
|
||||
<citations>
|
||||
<response_style>
|
||||
- Clear and Concise: Avoid over-formatting unless requested
|
||||
- Natural Tone: Use paragraphs and prose, not bullet points by default
|
||||
- Action-Oriented: Focus on delivering results, not explaining processes
|
||||
</response_style>
|
||||
|
||||
{ordinary_qa_format_section}
|
||||
|
||||
<citations>
|
||||
**CRITICAL: Always include citations when using web search results**
|
||||
|
||||
- **When to Use**: MANDATORY after web_search, web_fetch, or any external information source
|
||||
@@ -1374,7 +1388,7 @@ def build_report_structure_setup_prompt(
|
||||
"""
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
*,
|
||||
@@ -1386,10 +1400,11 @@ def apply_prompt_template(
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
) -> str:
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
is_scheduled_run: bool = False,
|
||||
) -> str:
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
@@ -1467,7 +1482,7 @@ Writing mode is enabled for this run.
|
||||
|
||||
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
|
||||
network_policy_section = _build_network_policy_section()
|
||||
rag_mode_section = (
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
@@ -1480,10 +1495,15 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
# Format the prompt with dynamic skills and memory
|
||||
else ""
|
||||
)
|
||||
ordinary_qa_format_section = (
|
||||
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION
|
||||
if not is_bootstrap and not writing_mode and not is_notebook_mode and not is_scheduled_run
|
||||
else ""
|
||||
)
|
||||
|
||||
# Format the prompt with dynamic skills and memory
|
||||
prompt = SYSTEM_PROMPT_TEMPLATE.format(
|
||||
agent_name=agent_name or storage_agent_id or "CM 助手",
|
||||
soul=get_agent_soul(storage_agent_id),
|
||||
@@ -1496,9 +1516,10 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
network_policy_section=network_policy_section,
|
||||
subagent_section=subagent_section,
|
||||
subagent_reminder=subagent_reminder,
|
||||
subagent_thinking=subagent_thinking,
|
||||
acp_section=acp_and_mounts_section,
|
||||
)
|
||||
subagent_thinking=subagent_thinking,
|
||||
acp_section=acp_and_mounts_section,
|
||||
ordinary_qa_format_section=ordinary_qa_format_section,
|
||||
)
|
||||
|
||||
prompt = prompt + rag_mode_section + _current_time_context()
|
||||
|
||||
|
||||
+104
-15
@@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
|
||||
No FastAPI or HTTP dependencies — pure utility functions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File extensions that should be converted to markdown
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".doc",
|
||||
".docx",
|
||||
}
|
||||
".docx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
}
|
||||
|
||||
# MarkItDown reads modern OOXML files directly. LibreOffice is used as a
|
||||
# compatibility bridge for legacy binary Office files and OpenDocument files,
|
||||
# which are common in offline/intranet deployments but are not handled
|
||||
# consistently by the pure-Python readers.
|
||||
_LIBREOFFICE_TARGETS = {
|
||||
".doc": ".docx",
|
||||
".ppt": ".pptx",
|
||||
".xls": ".xlsx",
|
||||
".odt": ".docx",
|
||||
".odp": ".pptx",
|
||||
".ods": ".xlsx",
|
||||
}
|
||||
_LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
|
||||
# Files larger than this threshold are converted in a background thread.
|
||||
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
|
||||
@@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
"""Convert any supported file to markdown text using MarkItDown."""
|
||||
from markitdown import MarkItDown
|
||||
|
||||
md = MarkItDown()
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
|
||||
"""Normalize a legacy Office/OpenDocument file, then extract its text.
|
||||
|
||||
Each conversion gets an isolated LibreOffice user profile. This avoids
|
||||
the profile-lock conflict that otherwise appears when several uploads are
|
||||
converted concurrently in the same sandbox container.
|
||||
"""
|
||||
target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower())
|
||||
if target_suffix is None:
|
||||
return None
|
||||
|
||||
executable = shutil.which("soffice") or shutil.which("libreoffice")
|
||||
if executable is None:
|
||||
logger.warning(
|
||||
"LibreOffice is unavailable; trying MarkItDown directly for %s",
|
||||
file_path.name,
|
||||
)
|
||||
return None
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary:
|
||||
temp_root = Path(temporary).resolve()
|
||||
output_dir = temp_root / "output"
|
||||
profile_dir = temp_root / "profile"
|
||||
output_dir.mkdir()
|
||||
profile_dir.mkdir()
|
||||
command = [
|
||||
executable,
|
||||
"--headless",
|
||||
"--nologo",
|
||||
"--nodefault",
|
||||
"--nolockcheck",
|
||||
"--nofirststartwizard",
|
||||
f"-env:UserInstallation={profile_dir.as_uri()}",
|
||||
"--convert-to",
|
||||
target_suffix.lstrip("."),
|
||||
"--outdir",
|
||||
str(output_dir),
|
||||
str(file_path.resolve()),
|
||||
]
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_LIBREOFFICE_TIMEOUT_SECONDS,
|
||||
env={**os.environ, "HOME": str(temp_root)},
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
logger.exception("LibreOffice failed to normalize %s", file_path.name)
|
||||
return None
|
||||
|
||||
converted_path = output_dir / f"{file_path.stem}{target_suffix}"
|
||||
if completed.returncode != 0 or not converted_path.is_file():
|
||||
logger.error(
|
||||
"LibreOffice could not normalize %s (exit=%s, stderr=%s)",
|
||||
file_path.name,
|
||||
completed.returncode,
|
||||
completed.stderr.strip()[-1000:],
|
||||
)
|
||||
return None
|
||||
return _convert_with_markitdown(converted_path)
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
|
||||
if normalized_office_text is not None:
|
||||
return normalized_office_text
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
# Try pymupdf4llm first (auto or explicit)
|
||||
pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path)
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline runtime smoke test for the LocalSandbox Office toolchain."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def require_command(name: str) -> str:
|
||||
path = shutil.which(name)
|
||||
if path is None:
|
||||
raise RuntimeError(f"missing command: {name}")
|
||||
return path
|
||||
|
||||
|
||||
def assert_contains(text: str, expected: str, label: str) -> None:
|
||||
normalized = text.replace("\\_", "_")
|
||||
if expected not in normalized:
|
||||
raise RuntimeError(f"{label} conversion lost test text: {text[:300]!r}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
from docx import Document
|
||||
from markitdown import MarkItDown
|
||||
from openpyxl import Workbook
|
||||
from pptx import Presentation
|
||||
|
||||
soffice = require_command("soffice")
|
||||
require_command("pandoc")
|
||||
require_command("pdftotext")
|
||||
require_command("file")
|
||||
|
||||
subprocess.run(
|
||||
[soffice, "--headless", "--version"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="deerflow-office-selfcheck-") as temporary:
|
||||
root = Path(temporary)
|
||||
|
||||
word_path = root / "word-test.docx"
|
||||
document = Document()
|
||||
document.add_heading("Word 自检", level=1)
|
||||
document.add_paragraph("DEERFLOW_WORD_OK")
|
||||
document.save(word_path)
|
||||
|
||||
excel_path = root / "excel-test.xlsx"
|
||||
workbook = Workbook()
|
||||
worksheet = workbook.active
|
||||
worksheet.title = "自检"
|
||||
worksheet.append(["状态", "DEERFLOW_EXCEL_OK"])
|
||||
workbook.save(excel_path)
|
||||
|
||||
ppt_path = root / "ppt-test.pptx"
|
||||
presentation = Presentation()
|
||||
slide = presentation.slides.add_slide(presentation.slide_layouts[5])
|
||||
slide.shapes.title.text = "DEERFLOW_PPT_OK"
|
||||
presentation.save(ppt_path)
|
||||
|
||||
converter = MarkItDown()
|
||||
assert_contains(converter.convert(word_path).text_content, "DEERFLOW_WORD_OK", "Word")
|
||||
assert_contains(converter.convert(excel_path).text_content, "DEERFLOW_EXCEL_OK", "Excel")
|
||||
assert_contains(converter.convert(ppt_path).text_content, "DEERFLOW_PPT_OK", "PowerPoint")
|
||||
|
||||
print("OFFICE_RUNTIME_SELF_CHECK=PASS")
|
||||
print("支持:DOC/DOCX、XLS/XLSX、PPT/PPTX、ODT/ODS/ODP、PDF")
|
||||
print("组件:MarkItDown、python-docx、openpyxl、python-pptx、LibreOffice、Pandoc、中文字体")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -68,3 +68,29 @@ def test_es_query_routing_is_injected_into_lead_and_allowed_custom_agent_prompts
|
||||
assert "自定义 ES 路由规则" in lead_prompt
|
||||
assert '<skill_routing skill="es_query">' in allowed_custom_prompt
|
||||
assert '<skill_routing skill="es_query">' not in denied_custom_prompt
|
||||
|
||||
|
||||
def test_formal_markdown_rules_are_scoped_to_ordinary_qa(monkeypatch) -> None:
|
||||
config = _app_config(enabled=False)
|
||||
monkeypatch.setattr(prompt_module, "_get_memory_context", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "get_agent_soul", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "get_skills_prompt_section", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "_build_acp_section", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "_build_custom_mounts_section", lambda *args, **kwargs: "")
|
||||
monkeypatch.setattr(prompt_module, "_current_time_context", lambda: "")
|
||||
|
||||
ordinary_prompt = prompt_module.apply_prompt_template(app_config=config)
|
||||
writing_prompt = prompt_module.apply_prompt_template(app_config=config, writing_mode=True)
|
||||
notebook_prompt = prompt_module.apply_prompt_template(app_config=config, is_notebook_mode=True)
|
||||
scheduled_prompt = prompt_module.apply_prompt_template(app_config=config, is_scheduled_run=True)
|
||||
|
||||
assert '<ordinary_qa_markdown_format>' in ordinary_prompt
|
||||
assert "## 一、标题" in ordinary_prompt
|
||||
assert "### (一) 标题" in ordinary_prompt
|
||||
assert "#### 1. 标题" in ordinary_prompt
|
||||
assert "2.1.1" in ordinary_prompt
|
||||
assert "无序圆点列表" in ordinary_prompt
|
||||
assert '<ordinary_qa_markdown_format>' not in writing_prompt
|
||||
assert '<ordinary_qa_markdown_format>' not in notebook_prompt
|
||||
assert '<ordinary_qa_markdown_format>' not in scheduled_prompt
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow.utils import file_conversion
|
||||
|
||||
|
||||
def test_legacy_word_is_normalized_before_markitdown(monkeypatch, tmp_path: Path) -> None:
|
||||
source = tmp_path / "legacy.doc"
|
||||
source.write_bytes(b"legacy")
|
||||
|
||||
monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: "/usr/bin/soffice")
|
||||
monkeypatch.setattr(
|
||||
file_conversion,
|
||||
"_convert_with_markitdown",
|
||||
lambda path: f"converted:{path.suffix}",
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
output_dir = Path(command[command.index("--outdir") + 1])
|
||||
(output_dir / "legacy.docx").write_bytes(b"normalized")
|
||||
return SimpleNamespace(returncode=0, stderr="")
|
||||
|
||||
monkeypatch.setattr(file_conversion.subprocess, "run", fake_run)
|
||||
|
||||
assert file_conversion._do_convert(source, "auto") == "converted:.docx"
|
||||
|
||||
|
||||
def test_modern_office_file_does_not_start_libreoffice(monkeypatch, tmp_path: Path) -> None:
|
||||
source = tmp_path / "modern.pptx"
|
||||
source.write_bytes(b"modern")
|
||||
monkeypatch.setattr(
|
||||
file_conversion,
|
||||
"_convert_with_markitdown",
|
||||
lambda path: f"direct:{path.suffix}",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
file_conversion.subprocess,
|
||||
"run",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected LibreOffice call")),
|
||||
)
|
||||
|
||||
assert file_conversion._do_convert(source, "auto") == "direct:.pptx"
|
||||
|
||||
|
||||
def test_legacy_office_falls_back_when_libreoffice_is_missing(monkeypatch, tmp_path: Path) -> None:
|
||||
source = tmp_path / "legacy.xls"
|
||||
source.write_bytes(b"legacy")
|
||||
monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: None)
|
||||
monkeypatch.setattr(
|
||||
file_conversion,
|
||||
"_convert_with_markitdown",
|
||||
lambda path: f"fallback:{path.suffix}",
|
||||
)
|
||||
|
||||
assert file_conversion._do_convert(source, "auto") == "fallback:.xls"
|
||||
@@ -41,8 +41,10 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
|
||||
## 一、一级标题
|
||||
这是正文,用于验证首行缩进、两端对齐和固定行距。
|
||||
|
||||
### (一)二级标题
|
||||
#### 1. 三级标题
|
||||
### 2.1 二级标题
|
||||
#### 2.1.1 三级标题
|
||||
|
||||
---
|
||||
|
||||
表1 测试表格
|
||||
| 项目 | 数值 |
|
||||
@@ -77,8 +79,11 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
|
||||
assert '<w:pStyle w:val="DocumentTitle"/>' in document
|
||||
assert "方正小标宋简体" in document
|
||||
assert "一、一级标题" not in document
|
||||
assert "(一)二级标题" not in document
|
||||
assert "1. 三级标题" not in document
|
||||
assert "2.1 二级标题" not in document
|
||||
assert "2.1.1 三级标题" not in document
|
||||
assert "────────" not in document
|
||||
assert "二级标题" in document
|
||||
assert "三级标题" in document
|
||||
|
||||
assert 'w:line="579" w:lineRule="exact"' in styles
|
||||
assert '<w:ind w:left="0" w:hanging="0" w:firstLineChars="200"/>' in styles
|
||||
@@ -86,11 +91,13 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa
|
||||
assert 'w:val="44"' in styles
|
||||
assert "仿宋" in styles and 'w:val="32"' in styles
|
||||
assert "宋体" in styles and 'w:val="28"' in styles
|
||||
assert styles.count('w:firstLineChars="200"') >= 6
|
||||
|
||||
assert '<w:numFmt w:val="chineseCounting"/>' in numbering
|
||||
assert '<w:lvlText w:val="%1、"/>' in numbering
|
||||
assert '<w:lvlText w:val="(%2)"/>' in numbering
|
||||
assert '<w:lvlText w:val="(%2)"/>' in numbering
|
||||
assert '<w:lvlText w:val="%3."/>' in numbering
|
||||
assert numbering.count('w:firstLineChars="200"') == 3
|
||||
assert '<w:evenAndOddHeaders/>' in settings
|
||||
assert '<w:updateFields w:val="true"/>' in settings
|
||||
|
||||
@@ -114,3 +121,20 @@ def test_export_rejects_wrong_font(tmp_path: Path) -> None:
|
||||
|
||||
with pytest.raises(word_export.WordExportFontError):
|
||||
word_export.build_markdown_docx("正文", font_path=wrong_font)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
[
|
||||
("一、一级标题", "一级标题"),
|
||||
("(一) 二级标题", "二级标题"),
|
||||
("(一)二级标题", "二级标题"),
|
||||
("1. 三级标题", "三级标题"),
|
||||
("2.1 二级标题", "二级标题"),
|
||||
("2.1.1 三级标题", "三级标题"),
|
||||
("2.1.1. 三级标题", "三级标题"),
|
||||
("正文标题", "正文标题"),
|
||||
],
|
||||
)
|
||||
def test_manual_heading_number_is_removed_as_one_complete_prefix(source: str, expected: str) -> None:
|
||||
assert word_export._strip_manual_heading_number(source) == expected
|
||||
Reference in new issue
Block a user