feat: standardize Word export and Office conversion
This commit is contained in:
1 parent
4ce2f3a502
commit
485a8fa9d9
12 files changed
+410
-60
No files matched your search
+5
-4
@@ -686,10 +686,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig):
|
||||
available_skills=prompt_available_skills,
|
||||
app_config=resolved_app_config,
|
||||
memory_injection_enabled=memory_injection_enabled,
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
)
|
||||
rag_mode_enabled=rag_mode_enabled,
|
||||
is_notebook_mode=is_notebook_mode,
|
||||
notebook_source_ids=notebook_source_ids,
|
||||
is_scheduled_run=bool(cfg.get("is_scheduled_run", False)),
|
||||
)
|
||||
|
||||
return create_agent(
|
||||
model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, force_disable_thinking=thinking_force_disabled, app_config=resolved_app_config),
|
||||
|
||||
+42
-21
@@ -486,7 +486,19 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p
|
||||
</subagent_system>"""
|
||||
|
||||
|
||||
SYSTEM_PROMPT_TEMPLATE = """
|
||||
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION = """
|
||||
<ordinary_qa_markdown_format>
|
||||
当普通问答需要输出带章节结构的较长内容时,必须采用以下 Markdown 规范,以便下载为 Word 后得到正确的自动编号和公文标题格式:
|
||||
- 文档主标题使用 `# 主标题`;主标题只写标题文字,不添加章节编号。
|
||||
- 一级章节使用 `## 一、标题`,二级章节使用 `### (一) 标题`,三级章节使用 `#### 1. 标题`。二级编号必须使用半角括号 `(一)`。
|
||||
- 不要在标题中使用 `2.1`、`2.1.1` 等多级阿拉伯数字编号;章节编号直接按上述三种形式书写。
|
||||
- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表;并列内容优先改写为完整自然段,确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。
|
||||
- 简短问答不必为了套用格式而强行添加标题;本规范只约束实际使用的标题和列表。
|
||||
</ordinary_qa_markdown_format>
|
||||
"""
|
||||
|
||||
|
||||
SYSTEM_PROMPT_TEMPLATE = """
|
||||
<role>
|
||||
You are {agent_name}, an internal super agent.
|
||||
</role>
|
||||
@@ -612,13 +624,15 @@ You: "Deploying to staging..." [proceed]
|
||||
{acp_section}
|
||||
</working_directory>
|
||||
|
||||
<response_style>
|
||||
- Clear and Concise: Avoid over-formatting unless requested
|
||||
- Natural Tone: Use paragraphs and prose, not bullet points by default
|
||||
- Action-Oriented: Focus on delivering results, not explaining processes
|
||||
</response_style>
|
||||
|
||||
<citations>
|
||||
<response_style>
|
||||
- Clear and Concise: Avoid over-formatting unless requested
|
||||
- Natural Tone: Use paragraphs and prose, not bullet points by default
|
||||
- Action-Oriented: Focus on delivering results, not explaining processes
|
||||
</response_style>
|
||||
|
||||
{ordinary_qa_format_section}
|
||||
|
||||
<citations>
|
||||
**CRITICAL: Always include citations when using web search results**
|
||||
|
||||
- **When to Use**: MANDATORY after web_search, web_fetch, or any external information source
|
||||
@@ -1374,7 +1388,7 @@ def build_report_structure_setup_prompt(
|
||||
"""
|
||||
|
||||
|
||||
def apply_prompt_template(
|
||||
def apply_prompt_template(
|
||||
subagent_enabled: bool = False,
|
||||
max_concurrent_subagents: int = 3,
|
||||
*,
|
||||
@@ -1386,10 +1400,11 @@ def apply_prompt_template(
|
||||
is_bootstrap: bool = False,
|
||||
app_config: AppConfig | None = None,
|
||||
memory_injection_enabled: bool = True,
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
) -> str:
|
||||
rag_mode_enabled: bool = False,
|
||||
is_notebook_mode: bool = False,
|
||||
notebook_source_ids: list[int] | None = None,
|
||||
is_scheduled_run: bool = False,
|
||||
) -> str:
|
||||
storage_agent_id = agent_id if agent_id is not None else agent_name
|
||||
|
||||
# Get memory context
|
||||
@@ -1467,7 +1482,7 @@ Writing mode is enabled for this run.
|
||||
|
||||
# Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set).
|
||||
network_policy_section = _build_network_policy_section()
|
||||
rag_mode_section = (
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
@@ -1480,10 +1495,15 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
# Format the prompt with dynamic skills and memory
|
||||
else ""
|
||||
)
|
||||
ordinary_qa_format_section = (
|
||||
_ORDINARY_QA_MARKDOWN_FORMAT_SECTION
|
||||
if not is_bootstrap and not writing_mode and not is_notebook_mode and not is_scheduled_run
|
||||
else ""
|
||||
)
|
||||
|
||||
# Format the prompt with dynamic skills and memory
|
||||
prompt = SYSTEM_PROMPT_TEMPLATE.format(
|
||||
agent_name=agent_name or storage_agent_id or "CM 助手",
|
||||
soul=get_agent_soul(storage_agent_id),
|
||||
@@ -1496,9 +1516,10 @@ When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
network_policy_section=network_policy_section,
|
||||
subagent_section=subagent_section,
|
||||
subagent_reminder=subagent_reminder,
|
||||
subagent_thinking=subagent_thinking,
|
||||
acp_section=acp_and_mounts_section,
|
||||
)
|
||||
subagent_thinking=subagent_thinking,
|
||||
acp_section=acp_and_mounts_section,
|
||||
ordinary_qa_format_section=ordinary_qa_format_section,
|
||||
)
|
||||
|
||||
prompt = prompt + rag_mode_section + _current_time_context()
|
||||
|
||||
|
||||
+104
-15
@@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569).
|
||||
No FastAPI or HTTP dependencies — pure utility functions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File extensions that should be converted to markdown
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
CONVERTIBLE_EXTENSIONS = {
|
||||
".pdf",
|
||||
".ppt",
|
||||
".pptx",
|
||||
".xls",
|
||||
".xlsx",
|
||||
".doc",
|
||||
".docx",
|
||||
}
|
||||
".docx",
|
||||
".odt",
|
||||
".ods",
|
||||
".odp",
|
||||
}
|
||||
|
||||
# MarkItDown reads modern OOXML files directly. LibreOffice is used as a
|
||||
# compatibility bridge for legacy binary Office files and OpenDocument files,
|
||||
# which are common in offline/intranet deployments but are not handled
|
||||
# consistently by the pure-Python readers.
|
||||
_LIBREOFFICE_TARGETS = {
|
||||
".doc": ".docx",
|
||||
".ppt": ".pptx",
|
||||
".xls": ".xlsx",
|
||||
".odt": ".docx",
|
||||
".odp": ".pptx",
|
||||
".ods": ".xlsx",
|
||||
}
|
||||
_LIBREOFFICE_TIMEOUT_SECONDS = 120
|
||||
|
||||
# Files larger than this threshold are converted in a background thread.
|
||||
# Small files complete in < 1s synchronously; spawning a thread adds unnecessary
|
||||
@@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
def _convert_with_markitdown(file_path: Path) -> str:
|
||||
"""Convert any supported file to markdown text using MarkItDown."""
|
||||
from markitdown import MarkItDown
|
||||
|
||||
md = MarkItDown()
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
return md.convert(str(file_path)).text_content
|
||||
|
||||
|
||||
def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None:
|
||||
"""Normalize a legacy Office/OpenDocument file, then extract its text.
|
||||
|
||||
Each conversion gets an isolated LibreOffice user profile. This avoids
|
||||
the profile-lock conflict that otherwise appears when several uploads are
|
||||
converted concurrently in the same sandbox container.
|
||||
"""
|
||||
target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower())
|
||||
if target_suffix is None:
|
||||
return None
|
||||
|
||||
executable = shutil.which("soffice") or shutil.which("libreoffice")
|
||||
if executable is None:
|
||||
logger.warning(
|
||||
"LibreOffice is unavailable; trying MarkItDown directly for %s",
|
||||
file_path.name,
|
||||
)
|
||||
return None
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary:
|
||||
temp_root = Path(temporary).resolve()
|
||||
output_dir = temp_root / "output"
|
||||
profile_dir = temp_root / "profile"
|
||||
output_dir.mkdir()
|
||||
profile_dir.mkdir()
|
||||
command = [
|
||||
executable,
|
||||
"--headless",
|
||||
"--nologo",
|
||||
"--nodefault",
|
||||
"--nolockcheck",
|
||||
"--nofirststartwizard",
|
||||
f"-env:UserInstallation={profile_dir.as_uri()}",
|
||||
"--convert-to",
|
||||
target_suffix.lstrip("."),
|
||||
"--outdir",
|
||||
str(output_dir),
|
||||
str(file_path.resolve()),
|
||||
]
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=_LIBREOFFICE_TIMEOUT_SECONDS,
|
||||
env={**os.environ, "HOME": str(temp_root)},
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
logger.exception("LibreOffice failed to normalize %s", file_path.name)
|
||||
return None
|
||||
|
||||
converted_path = output_dir / f"{file_path.stem}{target_suffix}"
|
||||
if completed.returncode != 0 or not converted_path.is_file():
|
||||
logger.error(
|
||||
"LibreOffice could not normalize %s (exit=%s, stderr=%s)",
|
||||
file_path.name,
|
||||
completed.returncode,
|
||||
completed.stderr.strip()[-1000:],
|
||||
)
|
||||
return None
|
||||
return _convert_with_markitdown(converted_path)
|
||||
|
||||
|
||||
def _do_convert(file_path: Path, pdf_converter: str) -> str:
|
||||
"""Synchronous conversion — called directly or via asyncio.to_thread.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file.
|
||||
pdf_converter: "auto" | "pymupdf4llm" | "markitdown"
|
||||
"""
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
is_pdf = file_path.suffix.lower() == ".pdf"
|
||||
|
||||
normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path)
|
||||
if normalized_office_text is not None:
|
||||
return normalized_office_text
|
||||
|
||||
if is_pdf and pdf_converter != "markitdown":
|
||||
# Try pymupdf4llm first (auto or explicit)
|
||||
pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path)
|
||||
|
||||
|
||||
Reference in new issue
Block a user