From 485a8fa9d9b74b46ff486ac0a0194b409ed5a70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Thu, 27 Aug 2026 00:08:52 +0800 Subject: [PATCH 1/2] feat: standardize Word export and Office conversion --- offline-backend-20260512/backend/CLAUDE.md | 15 ++- .../backend/Dockerfile.backend-offline | 18 ++- offline-backend-20260512/backend/README.md | 12 +- .../backend/app/gateway/word_export.py | 19 ++- .../backend/docs/FILE_UPLOAD.md | 20 ++- .../deerflow/agents/lead_agent/agent.py | 9 +- .../deerflow/agents/lead_agent/prompt.py | 63 ++++++---- .../harness/deerflow/utils/file_conversion.py | 119 +++++++++++++++--- .../scripts/office_runtime_selfcheck.py | 78 ++++++++++++ .../tests/test_es_query_routing_prompt.py | 26 ++++ .../tests/test_file_conversion_office.py | 57 +++++++++ .../backend/tests/test_word_export.py | 34 ++++- 12 files changed, 410 insertions(+), 60 deletions(-) create mode 100644 offline-backend-20260512/backend/scripts/office_runtime_selfcheck.py create mode 100644 offline-backend-20260512/backend/tests/test_file_conversion_office.py diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index b15e5a31..3371d9c1 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -324,7 +324,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api | **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail | | **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` | | **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types | -| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`. | +| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering (`一、` / `(一)` / `1.`), two-character first-line indent on heading levels 1–3, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. Before numbering it removes complete manual heading prefixes (including `2.1` / `2.1.1`) and drops Markdown horizontal rules from Word output. The default ordinary-Q&A Lead Agent prompt emits the matching Markdown heading forms and avoids unordered bullet lists; writing setup, writing mode, notebook, scheduled runs, and SOUL-driven custom agents keep their independent prompts. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`, `tests/test_es_query_routing_prompt.py`. | | **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. | | **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. | | **Suggestions** (`/api/threads/{id}/suggestions`) | `POST /` - generate follow-up questions; rich list/block model content is normalized before JSON parsing | @@ -1611,3 +1611,16 @@ write, log the exception, complete the rewrite with `versionSnapshotFailed: true`, and do not emit a user-facing failure or restore the original. The sandbox uses that signal only to enable its ordinary manual save fallback. + +### Offline local-sandbox Office runtime + +The DeerFlow-only offline deployment uses `LocalSandboxProvider`, so document +dependencies belong in the backend image rather than a separate sandbox image. +`deploy/linux-docker-offline/Dockerfile.backend` installs python-docx, +LibreOffice Writer/Calc/Impress, Pandoc, Poppler, and CJK fonts in addition to +the MarkItDown Office extras. `file_conversion.py` normalizes legacy +DOC/XLS/PPT and ODT/ODS/ODP through a per-conversion LibreOffice user profile +before MarkItDown extraction, avoiding concurrent profile locks. The image +build and `OFFLINE_INSTALL.sh --office-check` both run +`scripts/office_runtime_selfcheck.py`; that smoke test creates and re-parses +minimal DOCX/XLSX/PPTX files and does not need the application database. diff --git a/offline-backend-20260512/backend/Dockerfile.backend-offline b/offline-backend-20260512/backend/Dockerfile.backend-offline index bf562cf8..1a9d0bc4 100644 --- a/offline-backend-20260512/backend/Dockerfile.backend-offline +++ b/offline-backend-20260512/backend/Dockerfile.backend-offline @@ -60,7 +60,10 @@ RUN python -m venv /opt/venv \ && /opt/venv/bin/python -m pip install --no-cache-dir --upgrade -i "${UV_INDEX_URL}" pip setuptools wheel \ && sed -E '/--hash=/d; s/[[:space:]]+\\$//; s|^-e[[:space:]]+./packages/harness$|./packages/harness|' requirements-offline.txt > /tmp/requirements-docker.txt \ && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" -r /tmp/requirements-docker.txt \ - && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" "asyncmy>=0.2.11" + && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" \ + "asyncmy>=0.2.11" \ + "python-docx==1.2.0" \ + && /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd; print("Office Python dependencies OK")' COPY . . @@ -92,9 +95,17 @@ RUN if [ -n "${APT_MIRROR}" ]; then \ ca-certificates \ curl \ docker.io \ + file \ + fontconfig \ + fonts-liberation \ + fonts-noto-cjk \ libmagic1 \ + libreoffice-calc \ + libreoffice-impress \ + libreoffice-writer \ nodejs \ npm \ + pandoc \ poppler-utils \ tini \ && rm -rf /var/lib/apt/lists/* @@ -102,6 +113,11 @@ RUN if [ -n "${APT_MIRROR}" ]; then \ COPY --from=builder /opt/venv /opt/venv COPY --from=builder /build/backend /app/backend +RUN /opt/venv/bin/python -c 'import docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, xlrd' \ + && command -v soffice >/dev/null \ + && command -v pandoc >/dev/null \ + && fc-list :lang=zh | grep -q . + EXPOSE 8001 ENTRYPOINT ["tini", "--"] diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index b3e23a78..bedd9bdf 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -156,7 +156,7 @@ FastAPI application providing REST endpoints for frontend integration: | `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. | | `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. | | `POST /api/ai-writing/sample/extract` | Extract text from Word/PDF/Markdown/TXT samples for AI-writing imitation mode | -| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. | +| `POST /api/writing/export/docx` | Generate the shared formal Word download from Markdown. Uses the standard-library OOXML builder in `app/gateway/word_export.py`, removes manual/compound heading prefixes and Markdown horizontal rules, applies the prescribed A4/margin/heading/body/table/footer profile, and embeds the server-bundled `方正小标宋简体.ttf`; the client computer does not need that font installed. | | `POST /cop/saveSuperiorTask` | TaskCOP compatibility API (DeerFlow Bearer/session authentication required): create a situation-overview task with a sequential four-digit id (`0001`, `0002`, ...) and return the legacy `{state, msg, data}` envelope | | `POST /api/taskcop/import/tasks` | TaskCOP compatibility API (authenticated): import tasks from a JSON array or `tasks`/`records` envelope; every row's supplied `id`/`taskId` is authoritative and a duplicate id completely replaces the stored task | | `DELETE /cop/tasks/{task_id}` | TaskCOP compatibility API (authenticated): delete one task and its task-scoped situation-report document | @@ -943,3 +943,13 @@ The final rewrite commit uses a deliberately short sibling temporary filename, which keeps long Windows artifact paths below the path-length limit. It also recreates a missing `outputs` parent before retrying, so transient sandbox-directory recreation cannot discard a completed rewrite. + +## Offline local-sandbox Office runtime + +The pure DeerFlow offline image treats the backend container as the default +`LocalSandboxProvider` runtime. It bundles MarkItDown, python-docx, openpyxl, +python-pptx, LibreOffice Headless, Pandoc, Poppler, and CJK fonts. Modern OOXML +uploads are parsed directly; legacy DOC/XLS/PPT and ODT/ODS/ODP files are +normalized through an isolated LibreOffice profile before Markdown extraction. +Run `./OFFLINE_INSTALL.sh --office-check` from the offline bundle to generate +and re-parse Word, Excel, and PowerPoint samples without requiring a database. diff --git a/offline-backend-20260512/backend/app/gateway/word_export.py b/offline-backend-20260512/backend/app/gateway/word_export.py index eb164d57..162ab2f8 100644 --- a/offline-backend-20260512/backend/app/gateway/word_export.py +++ b/offline-backend-20260512/backend/app/gateway/word_export.py @@ -214,7 +214,13 @@ def _strip_inline_markdown(text: str) -> str: def _strip_manual_heading_number(text: str) -> str: return re.sub( - r"^\s*(?:[一二三四五六七八九十百千]+、|[（(][一二三四五六七八九十百千]+[）)]|\d+[.、]|[（(]\d+[）)]|\d+[）)])\s*", + r"^\s*(?:" + r"[一二三四五六七八九十百千]+[、.．]" + r"|[（(]\s*[一二三四五六七八九十百千]+\s*[）)]" + r"|[（(]\s*\d+(?:[.．]\d+)*\s*[）)]" + r"|\d+(?:[.．]\d+)+(?:[.．、])?" + r"|\d+[.．、)）]" + r")\s*", "", text, ).strip() @@ -338,7 +344,6 @@ def _parse_markdown(markdown: str, explicit_title: str | None) -> tuple[str | No if re.fullmatch(r"\s*(?:---|\*\*\*|___)\s*", raw): flush_paragraph() - blocks.append(ParagraphBlock("horizontal_rule", "────────")) index += 1 continue @@ -584,8 +589,6 @@ def _blocks_xml(blocks: list[Block], relationships: _DocumentRelationships) -> s output.append(_paragraph_xml("QuoteText", _inline_xml(block.text, relationships))) elif block.kind == "image": output.append(_image_paragraph_xml(block.text, relationships)) - elif block.kind == "horizontal_rule": - output.append(_paragraph_xml("BodyText", _run_xml(block.text), alignment="center")) else: output.append(_paragraph_xml("BodyText", _inline_xml(block.text, relationships))) return "".join(output) @@ -677,6 +680,7 @@ def _styles_xml() -> str: font=HEADING_1_FONT_FAMILY, size=BODY_SIZE_HALF_POINTS, alignment="left", + first_line_chars=200, keep_next=True, keep_lines=True, outline_level=0, @@ -687,6 +691,7 @@ def _styles_xml() -> str: font=HEADING_2_FONT_FAMILY, size=BODY_SIZE_HALF_POINTS, alignment="left", + first_line_chars=200, keep_next=True, keep_lines=True, outline_level=1, @@ -700,6 +705,7 @@ def _styles_xml() -> str: font=BODY_FONT_FAMILY, size=BODY_SIZE_HALF_POINTS, alignment="left", + first_line_chars=200 if level == 3 else 0, keep_next=True, keep_lines=True, outline_level=level - 1, @@ -769,10 +775,11 @@ def _styles_xml() -> str: def _heading_level_xml(level: int, format_value: str, text: str, font: str) -> str: restart = f'' if level > 0 else "" + first_line_chars = ' w:firstLineChars="200"' if level <= 2 else "" return ( f'{restart}' f'' - '' + f'' f'{_font_run_properties(font, BODY_SIZE_HALF_POINTS)}' ) @@ -798,7 +805,7 @@ def _numbering_xml() -> str: heading_levels = "".join( [ _heading_level_xml(0, "chineseCounting", "%1、", HEADING_1_FONT_FAMILY), - _heading_level_xml(1, "chineseCounting", "（%2）", HEADING_2_FONT_FAMILY), + _heading_level_xml(1, "chineseCounting", "(%2)", HEADING_2_FONT_FAMILY), _heading_level_xml(2, "decimal", "%3.", BODY_FONT_FAMILY), _heading_level_xml(3, "decimal", "（%4）", BODY_FONT_FAMILY), _heading_level_xml(4, "decimal", "%5）", BODY_FONT_FAMILY), diff --git a/offline-backend-20260512/backend/docs/FILE_UPLOAD.md b/offline-backend-20260512/backend/docs/FILE_UPLOAD.md index b9d14afe..21ae17d4 100644 --- a/offline-backend-20260512/backend/docs/FILE_UPLOAD.md +++ b/offline-backend-20260512/backend/docs/FILE_UPLOAD.md @@ -106,13 +106,14 @@ DELETE /api/threads/{thread_id}/uploads/{filename} 以下格式在显式启用 `uploads.auto_convert_documents: true` 时会自动转换为 Markdown： - PDF (`.pdf`) -- PowerPoint (`.ppt`, `.pptx`) -- Excel (`.xls`, `.xlsx`) -- Word (`.doc`, `.docx`) +- PowerPoint (`.ppt`, `.pptx`) +- Excel (`.xls`, `.xlsx`) +- Word (`.doc`, `.docx`) +- OpenDocument (`.odt`, `.ods`, `.odp`) 转换后的 Markdown 文件会保存在同一目录下，文件名为原文件名 + `.md` 扩展名。 -默认情况下，自动转换是关闭的，以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。只有在受信任部署中明确接受此风险时，才应将 `uploads.auto_convert_documents` 设置为 `true`。 +通用部署可以关闭自动转换，以避免在网关主机上对不受信任的 Office/PDF 上传执行解析。本仓库当前 `config.yaml` 和纯 DeerFlow 离线部署包默认设置为 `uploads.auto_convert_documents: true`，用于受信任的内网部署。 ## Agent 集成 @@ -247,8 +248,15 @@ backend/.deer-flow/threads/ ### 依赖 -- `markitdown>=0.0.1a2` - 文档转换 -- `python-multipart>=0.0.20` - 文件上传处理 +- `markitdown>=0.0.1a2` - 文档转换 +- `python-multipart>=0.0.20` - 文件上传处理 +- 纯 DeerFlow 离线镜像还内置 `python-docx`、`openpyxl`、`python-pptx`、LibreOffice Headless、Pandoc、Poppler 和中文字体。旧版 DOC/XLS/PPT 及 ODT/ODS/ODP 会先经 LibreOffice 规范化，再由 MarkItDown 提取。 + +离线部署包可在不连接数据库的情况下运行依赖自检： + +```bash +./OFFLINE_INSTALL.sh --office-check +``` ## 故障排查 diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py index d7a761a1..733110f1 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -686,10 +686,11 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig): available_skills=prompt_available_skills, app_config=resolved_app_config, memory_injection_enabled=memory_injection_enabled, - rag_mode_enabled=rag_mode_enabled, - is_notebook_mode=is_notebook_mode, - notebook_source_ids=notebook_source_ids, - ) + rag_mode_enabled=rag_mode_enabled, + is_notebook_mode=is_notebook_mode, + notebook_source_ids=notebook_source_ids, + is_scheduled_run=bool(cfg.get("is_scheduled_run", False)), + ) return create_agent( model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, force_disable_thinking=thinking_force_disabled, app_config=resolved_app_config), diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 5d46c25c..849e10bf 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -486,7 +486,19 @@ task(description="Oracle Cloud analysis", prompt="...", subagent_type="general-p """ -SYSTEM_PROMPT_TEMPLATE = """ +_ORDINARY_QA_MARKDOWN_FORMAT_SECTION = """ + +当普通问答需要输出带章节结构的较长内容时，必须采用以下 Markdown 规范，以便下载为 Word 后得到正确的自动编号和公文标题格式： +- 文档主标题使用 `# 主标题`；主标题只写标题文字，不添加章节编号。 +- 一级章节使用 `## 一、标题`，二级章节使用 `### (一) 标题`，三级章节使用 `#### 1. 标题`。二级编号必须使用半角括号 `(一)`。 +- 不要在标题中使用 `2.1`、`2.1.1` 等多级阿拉伯数字编号；章节编号直接按上述三种形式书写。 +- 默认禁止使用以 `-`、`*`、`+` 或 `•` 开头的无序圆点列表；并列内容优先改写为完整自然段，确需逐项表达时使用有序列表。只有用户明确要求无序列表时才使用。 +- 简短问答不必为了套用格式而强行添加标题；本规范只约束实际使用的标题和列表。 + +""" + + +SYSTEM_PROMPT_TEMPLATE = """ You are {agent_name}, an internal super agent. @@ -612,13 +624,15 @@ You: "Deploying to staging..." [proceed] {acp_section} - -- Clear and Concise: Avoid over-formatting unless requested -- Natural Tone: Use paragraphs and prose, not bullet points by default -- Action-Oriented: Focus on delivering results, not explaining processes - - - + +- Clear and Concise: Avoid over-formatting unless requested +- Natural Tone: Use paragraphs and prose, not bullet points by default +- Action-Oriented: Focus on delivering results, not explaining processes + + +{ordinary_qa_format_section} + + **CRITICAL: Always include citations when using web search results** - **When to Use**: MANDATORY after web_search, web_fetch, or any external information source @@ -1374,7 +1388,7 @@ def build_report_structure_setup_prompt( """ -def apply_prompt_template( +def apply_prompt_template( subagent_enabled: bool = False, max_concurrent_subagents: int = 3, *, @@ -1386,10 +1400,11 @@ def apply_prompt_template( is_bootstrap: bool = False, app_config: AppConfig | None = None, memory_injection_enabled: bool = True, - rag_mode_enabled: bool = False, - is_notebook_mode: bool = False, - notebook_source_ids: list[int] | None = None, -) -> str: + rag_mode_enabled: bool = False, + is_notebook_mode: bool = False, + notebook_source_ids: list[int] | None = None, + is_scheduled_run: bool = False, +) -> str: storage_agent_id = agent_id if agent_id is not None else agent_name # Get memory context @@ -1467,7 +1482,7 @@ Writing mode is enabled for this run. # Build the offline-network policy block (empty unless DEER_FLOW_OFFLINE_MODE is set). network_policy_section = _build_network_policy_section() - rag_mode_section = ( + rag_mode_section = ( """ RAG/reference display mode is enabled. @@ -1480,10 +1495,15 @@ When a search, knowledge, skill, or MCP tool returns a result list: """ if rag_mode_enabled - else "" - ) - - # Format the prompt with dynamic skills and memory + else "" + ) + ordinary_qa_format_section = ( + _ORDINARY_QA_MARKDOWN_FORMAT_SECTION + if not is_bootstrap and not writing_mode and not is_notebook_mode and not is_scheduled_run + else "" + ) + + # Format the prompt with dynamic skills and memory prompt = SYSTEM_PROMPT_TEMPLATE.format( agent_name=agent_name or storage_agent_id or "CM 助手", soul=get_agent_soul(storage_agent_id), @@ -1496,9 +1516,10 @@ When a search, knowledge, skill, or MCP tool returns a result list: network_policy_section=network_policy_section, subagent_section=subagent_section, subagent_reminder=subagent_reminder, - subagent_thinking=subagent_thinking, - acp_section=acp_and_mounts_section, - ) + subagent_thinking=subagent_thinking, + acp_section=acp_and_mounts_section, + ordinary_qa_format_section=ordinary_qa_format_section, + ) prompt = prompt + rag_mode_section + _current_time_context() diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/utils/file_conversion.py b/offline-backend-20260512/backend/packages/harness/deerflow/utils/file_conversion.py index b49e487d..a1a13c18 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/utils/file_conversion.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/utils/file_conversion.py @@ -14,25 +14,46 @@ asyncio.to_thread() to avoid blocking the event loop (fixes #1569). No FastAPI or HTTP dependencies — pure utility functions. """ -import asyncio -import logging -import re -from pathlib import Path +import asyncio +import logging +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path from deerflow.config.app_config import get_app_config logger = logging.getLogger(__name__) # File extensions that should be converted to markdown -CONVERTIBLE_EXTENSIONS = { +CONVERTIBLE_EXTENSIONS = { ".pdf", ".ppt", ".pptx", ".xls", ".xlsx", ".doc", - ".docx", -} + ".docx", + ".odt", + ".ods", + ".odp", +} + +# MarkItDown reads modern OOXML files directly. LibreOffice is used as a +# compatibility bridge for legacy binary Office files and OpenDocument files, +# which are common in offline/intranet deployments but are not handled +# consistently by the pure-Python readers. +_LIBREOFFICE_TARGETS = { + ".doc": ".docx", + ".ppt": ".pptx", + ".xls": ".xlsx", + ".odt": ".docx", + ".odp": ".pptx", + ".ods": ".xlsx", +} +_LIBREOFFICE_TIMEOUT_SECONDS = 120 # Files larger than this threshold are converted in a background thread. # Small files complete in < 1s synchronously; spawning a thread adds unnecessary @@ -94,24 +115,92 @@ def _convert_pdf_with_pymupdf4llm(file_path: Path) -> str | None: return None -def _convert_with_markitdown(file_path: Path) -> str: +def _convert_with_markitdown(file_path: Path) -> str: """Convert any supported file to markdown text using MarkItDown.""" from markitdown import MarkItDown md = MarkItDown() - return md.convert(str(file_path)).text_content - - -def _do_convert(file_path: Path, pdf_converter: str) -> str: + return md.convert(str(file_path)).text_content + + +def _convert_with_libreoffice_then_markitdown(file_path: Path) -> str | None: + """Normalize a legacy Office/OpenDocument file, then extract its text. + + Each conversion gets an isolated LibreOffice user profile. This avoids + the profile-lock conflict that otherwise appears when several uploads are + converted concurrently in the same sandbox container. + """ + target_suffix = _LIBREOFFICE_TARGETS.get(file_path.suffix.lower()) + if target_suffix is None: + return None + + executable = shutil.which("soffice") or shutil.which("libreoffice") + if executable is None: + logger.warning( + "LibreOffice is unavailable; trying MarkItDown directly for %s", + file_path.name, + ) + return None + + with tempfile.TemporaryDirectory(prefix="deerflow-office-") as temporary: + temp_root = Path(temporary).resolve() + output_dir = temp_root / "output" + profile_dir = temp_root / "profile" + output_dir.mkdir() + profile_dir.mkdir() + command = [ + executable, + "--headless", + "--nologo", + "--nodefault", + "--nolockcheck", + "--nofirststartwizard", + f"-env:UserInstallation={profile_dir.as_uri()}", + "--convert-to", + target_suffix.lstrip("."), + "--outdir", + str(output_dir), + str(file_path.resolve()), + ] + try: + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=_LIBREOFFICE_TIMEOUT_SECONDS, + env={**os.environ, "HOME": str(temp_root)}, + ) + except (OSError, subprocess.TimeoutExpired): + logger.exception("LibreOffice failed to normalize %s", file_path.name) + return None + + converted_path = output_dir / f"{file_path.stem}{target_suffix}" + if completed.returncode != 0 or not converted_path.is_file(): + logger.error( + "LibreOffice could not normalize %s (exit=%s, stderr=%s)", + file_path.name, + completed.returncode, + completed.stderr.strip()[-1000:], + ) + return None + return _convert_with_markitdown(converted_path) + + +def _do_convert(file_path: Path, pdf_converter: str) -> str: """Synchronous conversion — called directly or via asyncio.to_thread. Args: file_path: Path to the file. pdf_converter: "auto" | "pymupdf4llm" | "markitdown" """ - is_pdf = file_path.suffix.lower() == ".pdf" - - if is_pdf and pdf_converter != "markitdown": + is_pdf = file_path.suffix.lower() == ".pdf" + + normalized_office_text = _convert_with_libreoffice_then_markitdown(file_path) + if normalized_office_text is not None: + return normalized_office_text + + if is_pdf and pdf_converter != "markitdown": # Try pymupdf4llm first (auto or explicit) pymupdf_text = _convert_pdf_with_pymupdf4llm(file_path) diff --git a/offline-backend-20260512/backend/scripts/office_runtime_selfcheck.py b/offline-backend-20260512/backend/scripts/office_runtime_selfcheck.py new file mode 100644 index 00000000..810c02a4 --- /dev/null +++ b/offline-backend-20260512/backend/scripts/office_runtime_selfcheck.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Offline runtime smoke test for the LocalSandbox Office toolchain.""" + +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def require_command(name: str) -> str: + path = shutil.which(name) + if path is None: + raise RuntimeError(f"missing command: {name}") + return path + + +def assert_contains(text: str, expected: str, label: str) -> None: + normalized = text.replace("\\_", "_") + if expected not in normalized: + raise RuntimeError(f"{label} conversion lost test text: {text[:300]!r}") + + +def main() -> int: + from docx import Document + from markitdown import MarkItDown + from openpyxl import Workbook + from pptx import Presentation + + soffice = require_command("soffice") + require_command("pandoc") + require_command("pdftotext") + require_command("file") + + subprocess.run( + [soffice, "--headless", "--version"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + + with tempfile.TemporaryDirectory(prefix="deerflow-office-selfcheck-") as temporary: + root = Path(temporary) + + word_path = root / "word-test.docx" + document = Document() + document.add_heading("Word 自检", level=1) + document.add_paragraph("DEERFLOW_WORD_OK") + document.save(word_path) + + excel_path = root / "excel-test.xlsx" + workbook = Workbook() + worksheet = workbook.active + worksheet.title = "自检" + worksheet.append(["状态", "DEERFLOW_EXCEL_OK"]) + workbook.save(excel_path) + + ppt_path = root / "ppt-test.pptx" + presentation = Presentation() + slide = presentation.slides.add_slide(presentation.slide_layouts[5]) + slide.shapes.title.text = "DEERFLOW_PPT_OK" + presentation.save(ppt_path) + + converter = MarkItDown() + assert_contains(converter.convert(word_path).text_content, "DEERFLOW_WORD_OK", "Word") + assert_contains(converter.convert(excel_path).text_content, "DEERFLOW_EXCEL_OK", "Excel") + assert_contains(converter.convert(ppt_path).text_content, "DEERFLOW_PPT_OK", "PowerPoint") + + print("OFFICE_RUNTIME_SELF_CHECK=PASS") + print("支持：DOC/DOCX、XLS/XLSX、PPT/PPTX、ODT/ODS/ODP、PDF") + print("组件：MarkItDown、python-docx、openpyxl、python-pptx、LibreOffice、Pandoc、中文字体") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py index 4837f9a0..ec6080b4 100644 --- a/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py +++ b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py @@ -68,3 +68,29 @@ def test_es_query_routing_is_injected_into_lead_and_allowed_custom_agent_prompts assert "自定义 ES 路由规则" in lead_prompt assert '' in allowed_custom_prompt assert '' not in denied_custom_prompt + + +def test_formal_markdown_rules_are_scoped_to_ordinary_qa(monkeypatch) -> None: + config = _app_config(enabled=False) + monkeypatch.setattr(prompt_module, "_get_memory_context", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "get_agent_soul", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "get_skills_prompt_section", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "_build_acp_section", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "_build_custom_mounts_section", lambda *args, **kwargs: "") + monkeypatch.setattr(prompt_module, "_current_time_context", lambda: "") + + ordinary_prompt = prompt_module.apply_prompt_template(app_config=config) + writing_prompt = prompt_module.apply_prompt_template(app_config=config, writing_mode=True) + notebook_prompt = prompt_module.apply_prompt_template(app_config=config, is_notebook_mode=True) + scheduled_prompt = prompt_module.apply_prompt_template(app_config=config, is_scheduled_run=True) + + assert '' in ordinary_prompt + assert "## 一、标题" in ordinary_prompt + assert "### (一) 标题" in ordinary_prompt + assert "#### 1. 标题" in ordinary_prompt + assert "2.1.1" in ordinary_prompt + assert "无序圆点列表" in ordinary_prompt + assert '' not in writing_prompt + assert '' not in notebook_prompt + assert '' not in scheduled_prompt diff --git a/offline-backend-20260512/backend/tests/test_file_conversion_office.py b/offline-backend-20260512/backend/tests/test_file_conversion_office.py new file mode 100644 index 00000000..0a079c09 --- /dev/null +++ b/offline-backend-20260512/backend/tests/test_file_conversion_office.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from deerflow.utils import file_conversion + + +def test_legacy_word_is_normalized_before_markitdown(monkeypatch, tmp_path: Path) -> None: + source = tmp_path / "legacy.doc" + source.write_bytes(b"legacy") + + monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: "/usr/bin/soffice") + monkeypatch.setattr( + file_conversion, + "_convert_with_markitdown", + lambda path: f"converted:{path.suffix}", + ) + + def fake_run(command, **_kwargs): + output_dir = Path(command[command.index("--outdir") + 1]) + (output_dir / "legacy.docx").write_bytes(b"normalized") + return SimpleNamespace(returncode=0, stderr="") + + monkeypatch.setattr(file_conversion.subprocess, "run", fake_run) + + assert file_conversion._do_convert(source, "auto") == "converted:.docx" + + +def test_modern_office_file_does_not_start_libreoffice(monkeypatch, tmp_path: Path) -> None: + source = tmp_path / "modern.pptx" + source.write_bytes(b"modern") + monkeypatch.setattr( + file_conversion, + "_convert_with_markitdown", + lambda path: f"direct:{path.suffix}", + ) + monkeypatch.setattr( + file_conversion.subprocess, + "run", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("unexpected LibreOffice call")), + ) + + assert file_conversion._do_convert(source, "auto") == "direct:.pptx" + + +def test_legacy_office_falls_back_when_libreoffice_is_missing(monkeypatch, tmp_path: Path) -> None: + source = tmp_path / "legacy.xls" + source.write_bytes(b"legacy") + monkeypatch.setattr(file_conversion.shutil, "which", lambda _name: None) + monkeypatch.setattr( + file_conversion, + "_convert_with_markitdown", + lambda path: f"fallback:{path.suffix}", + ) + + assert file_conversion._do_convert(source, "auto") == "fallback:.xls" diff --git a/offline-backend-20260512/backend/tests/test_word_export.py b/offline-backend-20260512/backend/tests/test_word_export.py index 9121c10c..c4611e63 100644 --- a/offline-backend-20260512/backend/tests/test_word_export.py +++ b/offline-backend-20260512/backend/tests/test_word_export.py @@ -41,8 +41,10 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa ## 一、一级标题 这是正文，用于验证首行缩进、两端对齐和固定行距。 -### （一）二级标题 -#### 1. 三级标题 +### 2.1 二级标题 +#### 2.1.1 三级标题 + +--- 表1 测试表格 | 项目 | 数值 | @@ -77,8 +79,11 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa assert '' in document assert "方正小标宋简体" in document assert "一、一级标题" not in document - assert "（一）二级标题" not in document - assert "1. 三级标题" not in document + assert "2.1 二级标题" not in document + assert "2.1.1 三级标题" not in document + assert "────────" not in document + assert "二级标题" in document + assert "三级标题" in document assert 'w:line="579" w:lineRule="exact"' in styles assert '' in styles @@ -86,11 +91,13 @@ def test_formal_docx_contains_required_layout_numbering_and_embedded_font(tmp_pa assert 'w:val="44"' in styles assert "仿宋" in styles and 'w:val="32"' in styles assert "宋体" in styles and 'w:val="28"' in styles + assert styles.count('w:firstLineChars="200"') >= 6 assert '' in numbering assert '' in numbering - assert '' in numbering + assert '' in numbering assert '' in numbering + assert numbering.count('w:firstLineChars="200"') == 3 assert '' in settings assert '' in settings @@ -114,3 +121,20 @@ def test_export_rejects_wrong_font(tmp_path: Path) -> None: with pytest.raises(word_export.WordExportFontError): word_export.build_markdown_docx("正文", font_path=wrong_font) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("一、一级标题", "一级标题"), + ("(一) 二级标题", "二级标题"), + ("（一）二级标题", "二级标题"), + ("1. 三级标题", "三级标题"), + ("2.1 二级标题", "二级标题"), + ("2.1.1 三级标题", "三级标题"), + ("2.1.1. 三级标题", "三级标题"), + ("正文标题", "正文标题"), + ], +) +def test_manual_heading_number_is_removed_as_one_complete_prefix(source: str, expected: str) -> None: + assert word_export._strip_manual_heading_number(source) == expected From 11326eeb1cb774a295773a4b7120fd1142731147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Thu, 27 Aug 2026 11:14:23 +0800 Subject: [PATCH 2/2] feat: improve skill uploads and agent runtime stability --- .../workspace/agents/skill-picker-sheet.tsx | 20 +- .../settings/skill-settings-page.tsx | 97 +++++--- frontend-web/src/core/i18n/locales/en-US.ts | 18 +- frontend-web/src/core/i18n/locales/types.ts | 9 +- frontend-web/src/core/i18n/locales/zh-CN.ts | 16 +- frontend-web/src/core/skills/api.ts | 49 ++-- frontend-web/src/core/skills/hooks.ts | 16 +- frontend-web/src/core/threads/hooks.ts | 71 ++++-- offline-backend-20260512/backend/CLAUDE.md | 8 +- offline-backend-20260512/backend/README.md | 6 +- .../backend/app/gateway/routers/skills.py | 143 +++++++++--- .../deerflow/agents/lead_agent/agent.py | 58 ++++- .../tests/test_es_query_routing_prompt.py | 40 ++++ .../tests/test_skills_install_upload.py | 220 ++++++++++++++++++ 14 files changed, 641 insertions(+), 130 deletions(-) diff --git a/frontend-web/src/components/workspace/agents/skill-picker-sheet.tsx b/frontend-web/src/components/workspace/agents/skill-picker-sheet.tsx index 55d4c5f9..f31fc960 100644 --- a/frontend-web/src/components/workspace/agents/skill-picker-sheet.tsx +++ b/frontend-web/src/components/workspace/agents/skill-picker-sheet.tsx @@ -165,10 +165,26 @@ export function SkillPickerSheet({ return; } try { - const validation = await validateUpload.mutateAsync(file); + const validation = await validateUpload.mutateAsync({ + file, + targetCategory: "custom", + }); if (validation.conflict) { + const owner = validation.owner_username ?? validation.owner_user_id; + if (validation.owned_by_other && owner) { + toast.error( + `技能「${validation.skill_name}」已由用户「${owner}」上传，不能覆盖。`, + ); + return; + } + if (validation.owned_by_me && validation.can_overwrite) { + toast.warning( + `您已上传过同名技能「${validation.skill_name}」，可到「技能」设置页重复上传并覆盖原版本。`, + ); + return; + } toast.warning( - `已存在同名技能「${validation.skill_name}」，请到「技能」设置页上传以选择是否覆盖。`, + `已存在同名技能「${validation.skill_name}」，当前不可覆盖。`, ); return; } diff --git a/frontend-web/src/components/workspace/settings/skill-settings-page.tsx b/frontend-web/src/components/workspace/settings/skill-settings-page.tsx index 42f92a24..d63df2c6 100644 --- a/frontend-web/src/components/workspace/settings/skill-settings-page.tsx +++ b/frontend-web/src/components/workspace/settings/skill-settings-page.tsx @@ -1201,7 +1201,24 @@ function InstallSkillDialog({ const validateMutation = useValidateSkillUpload(); const installMutation = useInstallSkillUpload(); const setNameZhMutation = useSetSkillNameZh(); - const generateNameZhMutation = useGenerateSkillNameZh(); + const generateNameZhMutation = useGenerateSkillNameZh(); + + const validateFile = useCallback( + async (selected: File, category: "custom" | "public") => { + setValidation(null); + setValidationError(null); + try { + const result = await validateMutation.mutateAsync({ + file: selected, + targetCategory: category, + }); + setValidation(result); + } catch (err) { + setValidationError(err instanceof Error ? err.message : String(err)); + } + }, + [validateMutation], + ); const reset = useCallback(() => { setFile(null); @@ -1240,19 +1257,12 @@ function InstallSkillDialog({ if (!lower.endsWith(".zip") && !lower.endsWith(".skill")) { toast.error(dialog.rejectInvalidExt); return; - } - setFile(selected); - setValidation(null); - setValidationError(null); - try { - const result = await validateMutation.mutateAsync(selected); - setValidation(result); - } catch (err) { - setValidationError(err instanceof Error ? err.message : String(err)); - } - }, - [dialog.rejectInvalidExt, validateMutation], - ); + } + setFile(selected); + await validateFile(selected, targetCategory); + }, + [dialog.rejectInvalidExt, targetCategory, validateFile], + ); const onDrop = useCallback( (e: React.DragEvent) => { @@ -1329,12 +1339,29 @@ function InstallSkillDialog({ } }, [dialog.nameZhSaved, installedName, nameZhDraft, onClose, reset, setNameZhMutation]); - const isPublicConflict = validation?.conflict && validation.conflict_kind === "public"; - const isCustomConflict = validation?.conflict && validation.conflict_kind === "custom"; - // Admin targeting public directory can overwrite an existing public skill. - const adminCanOverwritePublic = isAdmin && targetCategory === "public" && !!isPublicConflict; - const canInstall = !!validation && !validationError && !isCustomConflict && (adminCanOverwritePublic ? false : !isPublicConflict); - const canOverwrite = !!validation && !validationError && (!!isCustomConflict || adminCanOverwritePublic); + const isPublicConflict = validation?.conflict && validation.conflict_kind === "public"; + const isCustomConflict = validation?.conflict && validation.conflict_kind === "custom"; + // Admin targeting public directory can overwrite an existing public skill. + const adminCanOverwritePublic = isAdmin && targetCategory === "public" && !!isPublicConflict; + const canInstall = !!validation && !validationError && validation.can_install_as_is; + const canOverwrite = !!validation && !validationError && validation.can_overwrite; + const ownCustomCanOverwrite = !!isCustomConflict && !!validation?.owned_by_me && canOverwrite; + const otherCustomConflict = !!isCustomConflict && !!validation?.owned_by_other; + const customConflictMessage = !validation + ? "" + : otherCustomConflict + ? dialog.conflictOwnedByOther + .replace("{name}", validation.skill_name) + .replace( + "{owner}", + validation.owner_username ?? validation.owner_user_id ?? "—", + ) + : validation.owned_by_me + ? (ownCustomCanOverwrite + ? dialog.conflictOwnedByMe + : dialog.conflictOwnedByMeWrongTarget + ).replace("{name}", validation.skill_name) + : dialog.conflictCustom.replace("{name}", validation.skill_name); return ( !o && handleClose()}> @@ -1453,9 +1480,12 @@ function InstallSkillDialog({ setTargetCategory(cat)} + value={cat} + checked={targetCategory === cat} + onChange={() => { + setTargetCategory(cat); + if (file) void validateFile(file, cat); + }} className="accent-primary" /> {cat === "custom" ? dialog.categoryCustom : dialog.categoryPublic} @@ -1513,15 +1543,18 @@ function InstallSkillDialog({ )} - {isCustomConflict && ( -
- - - {desensitize( - dialog.conflictCustom.replace("{name}", validation.skill_name), - )} - -
+ {isCustomConflict && ( +
+ + + {desensitize(customConflictMessage)} + +
)} {isPublicConflict && (
diff --git a/frontend-web/src/core/i18n/locales/en-US.ts b/frontend-web/src/core/i18n/locales/en-US.ts index e8a7ad64..24bc3ff7 100644 --- a/frontend-web/src/core/i18n/locales/en-US.ts +++ b/frontend-web/src/core/i18n/locales/en-US.ts @@ -591,20 +591,26 @@ export const enUS: Translations = { chooseFile: "Choose file", validating: "Validating archive…", installing: "Installing…", - replace: "Replace existing", + replace: "Upload again and replace", install: "Install", cancel: "Cancel", nameLabel: "Name", descriptionLabel: "Description", licenseLabel: "License", toolsLabel: "Allowed tools", - warningsLabel: "Warnings", - conflictCustom: - "A custom skill named '{name}' is already installed. Use 'Replace existing' to overwrite (its enabled state will be kept).", + warningsLabel: "Warnings", + conflictOwnedByMe: + "You already uploaded a skill named '{name}'. You may upload it again and replace your existing copy (its enabled state will be kept).", + conflictOwnedByMeWrongTarget: + "The custom skill '{name}' belongs to you. Switch the target to 'Custom skill' to replace it.", + conflictOwnedByOther: + "Skill '{name}' was uploaded by user '{owner}' and cannot be replaced. Rename the skill before uploading.", + conflictCustom: + "A custom skill named '{name}' already exists, but its uploader could not be identified. Rename it to avoid overwriting someone else's skill.", conflictPublic: "A built-in public skill named '{name}' already exists and cannot be overwritten.", - conflictPublicAdminHint: - "A built-in public skill named '{name}' exists. Switch to 'Built-in' and click 'Replace existing' to overwrite.", + conflictPublicAdminHint: + "A built-in public skill named '{name}' exists. Click 'Upload again and replace' to overwrite it.", installSuccess: "Skill '{name}' installed.", installSuccessReplaced: "Skill '{name}' replaced.", rejectInvalidExt: "Only .zip or .skill files are accepted.", diff --git a/frontend-web/src/core/i18n/locales/types.ts b/frontend-web/src/core/i18n/locales/types.ts index 3cbdf9c0..0d33ce5d 100644 --- a/frontend-web/src/core/i18n/locales/types.ts +++ b/frontend-web/src/core/i18n/locales/types.ts @@ -525,9 +525,12 @@ export interface Translations { nameLabel: string; descriptionLabel: string; licenseLabel: string; - toolsLabel: string; - warningsLabel: string; - conflictCustom: string; + toolsLabel: string; + warningsLabel: string; + conflictOwnedByMe: string; + conflictOwnedByMeWrongTarget: string; + conflictOwnedByOther: string; + conflictCustom: string; conflictPublic: string; conflictPublicAdminHint: string; installSuccess: string; diff --git a/frontend-web/src/core/i18n/locales/zh-CN.ts b/frontend-web/src/core/i18n/locales/zh-CN.ts index 8610ac88..d9bff801 100644 --- a/frontend-web/src/core/i18n/locales/zh-CN.ts +++ b/frontend-web/src/core/i18n/locales/zh-CN.ts @@ -566,18 +566,24 @@ export const zhCN: Translations = { chooseFile: "选择文件", validating: "正在校验文件…", installing: "安装中…", - replace: "覆盖已有", + replace: "重复上传并覆盖", install: "安装", cancel: "取消", nameLabel: "名称", descriptionLabel: "描述", licenseLabel: "许可证", toolsLabel: "允许的工具", - warningsLabel: "警告", - conflictCustom: - "已存在同名自定义技能 “{name}”。点击「覆盖已有」可替换（原启用状态保留）。", + warningsLabel: "警告", + conflictOwnedByMe: + "您已上传过同名技能 “{name}”，可重复上传并覆盖原版本（原启用状态保留）。", + conflictOwnedByMeWrongTarget: + "同名技能 “{name}” 是您上传的自定义技能，请切换到「自定义技能」后覆盖。", + conflictOwnedByOther: + "技能 “{name}” 已由用户 “{owner}” 上传，不能覆盖，请修改技能名称后再上传。", + conflictCustom: + "已存在同名自定义技能 “{name}”，但无法确认上传用户，为避免误覆盖请修改技能名称。", conflictPublic: `存在同名内置技能 '{name}'，无法覆盖。`, - conflictPublicAdminHint: `存在同名内置技能 '{name}'。切换到「内置技能」并点击「覆盖已有」可替换。`, + conflictPublicAdminHint: `存在同名内置技能 '{name}'。点击「重复上传并覆盖」可替换。`, installSuccess: `技能 '{name}' 已安装。`, installSuccessReplaced: `技能 '{name}' 已替换。`, rejectInvalidExt: `只接受 .zip 或 .skill 文件。`, diff --git a/frontend-web/src/core/skills/api.ts b/frontend-web/src/core/skills/api.ts index 947a7395..0c6188ad 100644 --- a/frontend-web/src/core/skills/api.ts +++ b/frontend-web/src/core/skills/api.ts @@ -175,16 +175,24 @@ export interface PreviewSkillInstallResponse { warnings: string[]; } -export interface ValidateSkillUploadResponse { - ok: boolean; - skill_name: string; - description: string; - license: string | null; - allowed_tools: string[]; - conflict: boolean; - conflict_kind: "public" | "custom" | null; - warnings: string[]; -} +export interface ValidateSkillUploadResponse { + ok: boolean; + skill_name: string; + description: string; + license: string | null; + allowed_tools: string[]; + conflict: boolean; + conflict_kind: "public" | "custom" | null; + can_install_as_is: boolean; + can_overwrite: boolean; + owned_by_me: boolean; + owned_by_other: boolean; + builtin_conflict: boolean; + owner_username?: string | null; + owner_user_id?: string | null; + message: string; + warnings: string[]; +} export async function installSkill( request: InstallSkillRequest, @@ -235,13 +243,20 @@ async function parseDetailedError(response: Response, fallback: string): Promise return new Error(message); } -export async function validateSkillUpload(file: File): Promise { - const form = new FormData(); - form.append("file", file, file.name); - const response = await apiFetch(`${getBackendBaseURL()}/api/skills/validate`, { - method: "POST", - body: form, - }); +export async function validateSkillUpload( + file: File, + targetCategory: "public" | "custom" = "custom", +): Promise { + const form = new FormData(); + form.append("file", file, file.name); + const params = new URLSearchParams({ target_category: targetCategory }); + const response = await apiFetch( + `${getBackendBaseURL()}/api/skills/validate?${params.toString()}`, + { + method: "POST", + body: form, + }, + ); if (!response.ok) { throw await parseDetailedError(response, "Validation failed"); } diff --git a/frontend-web/src/core/skills/hooks.ts b/frontend-web/src/core/skills/hooks.ts index ad68d7d5..9da8298a 100644 --- a/frontend-web/src/core/skills/hooks.ts +++ b/frontend-web/src/core/skills/hooks.ts @@ -192,11 +192,17 @@ export function useSetSkillAlwaysOn() { }); } -export function useValidateSkillUpload() { - return useMutation({ - mutationFn: (file: File) => validateSkillUpload(file), - }); -} +export function useValidateSkillUpload() { + return useMutation({ + mutationFn: ({ + file, + targetCategory, + }: { + file: File; + targetCategory?: "public" | "custom"; + }) => validateSkillUpload(file, targetCategory), + }); +} export function useInstallSkillUpload() { const queryClient = useQueryClient(); diff --git a/frontend-web/src/core/threads/hooks.ts b/frontend-web/src/core/threads/hooks.ts index 1593f32d..011bdd3b 100644 --- a/frontend-web/src/core/threads/hooks.ts +++ b/frontend-web/src/core/threads/hooks.ts @@ -316,7 +316,7 @@ export async function stopActiveThreadBeforeClarificationReply( await new Promise((resolve) => setTimeout(resolve, 120)); } -function getStreamErrorMessage(error: unknown): string { +function getStreamErrorMessage(error: unknown): string { if (typeof error === "string" && error.trim()) { return error; } @@ -336,8 +336,21 @@ function getStreamErrorMessage(error: unknown): string { return nestedError; } } - return "Request failed."; -} + return "Request failed."; +} + +/** + * React production error #185 is the minified form of "Maximum update depth + * exceeded". It is a client render-loop guard, not a backend/model failure, + * so it must never be surfaced as the answer's stream error. + */ +function isReactMaximumUpdateDepthError(message: string): boolean { + return ( + /maximum update depth exceeded/i.test(message) || + /minified react error\s*#185\b/i.test(message) || + /react\.dev\/errors\/185\b/i.test(message) + ); +} const TERMINAL_RUN_STATUSES = new Set([ "success", @@ -670,11 +683,17 @@ export function useThreadStream({ runMetadataStorageRef.current = getRunMetadataStorage(); } - const thread = useStream({ - client: getAPIClient(isMock), - assistantId: assistantId ?? "lead_agent", - threadId: onStreamThreadId, - reconnectOnMount: runMetadataStorageRef.current + const thread = useStream({ + client: getAPIClient(isMock), + assistantId: assistantId ?? "lead_agent", + threadId: onStreamThreadId, + // The SDK's current implementation falls back to synchronous external- + // store notifications when this option is omitted. A network chunk can + // contain dozens of token events, causing nested React renders and the + // intermittent production-only React #185 error. Coalesce notifications + // to one visual update per frame; stream data itself is not dropped. + throttle: 16, + reconnectOnMount: runMetadataStorageRef.current ? () => runMetadataStorageRef.current! : false, fetchStateHistory: { limit: 1 }, @@ -820,10 +839,21 @@ export function useThreadStream({ toast(e.message); } }, - onError(error) { - markStreamActivity(); - const message = getStreamErrorMessage(error); - // Replacing a stalled local SSE reader aborts the in-flight fetch. That + onError(error) { + markStreamActivity(); + const message = getStreamErrorMessage(error); + // A synchronous SDK store notification used to let React's update-depth + // guard escape into the stream promise. Do not present that internal UI + // exception as a model failure. Reconcile/rejoin on the next task so the + // backend run can continue and the current answer is preserved. + if (isReactMaximumUpdateDepthError(message)) { + window.setTimeout(() => { + setStreamHoldOpen(true); + void confirmOrRejoinRef.current("error"); + }, 0); + return; + } + // Replacing a stalled local SSE reader aborts the in-flight fetch. That // is expected and must not look like a user-facing "network" failure. if (isAbortLikeError(error)) { return; @@ -1646,13 +1676,20 @@ export function useThreadStream({ thread.isLoading, ) ?? thread.values; - const mergedMessages = + const mergedMessages = Array.isArray(mergedValues?.messages) ? (mergedValues.messages as Message[]) - : thread.messages; - - const mergedThread = { - ...thread, + : thread.messages; + + const visibleThreadError = + thread.error && + isReactMaximumUpdateDepthError(getStreamErrorMessage(thread.error)) + ? undefined + : thread.error; + + const mergedThread = { + ...thread, + error: visibleThreadError, values: mergedValues, messages: optimisticMessages.length > 0 diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 3371d9c1..453bb26d 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -735,9 +735,13 @@ MySQL index script synchronized. `lead_agent.prompt._build_es_query_routing_section` injects the rule into the default lead-agent system prompt. Custom-agent-only prompts receive it only when their explicit skill allowlist contains `es_query`; disabling the switch - or leaving the prompt blank emits no block. Tests: + or leaving the prompt blank emits no block. Immediately before an ordinary Q&A + graph is created, `lead_agent.agent._log_es_query_routing_registration` checks + the exact configured block against the final system-prompt string and logs + `enabled`, allowlist eligibility, `registered`, character count, and a short + SHA-256 fingerprint (never the prompt body). Tests: `tests/test_es_query_routing_prompt.py`. -- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory +- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory. Direct uploads use the ownership-aware `POST /api/skills/validate?target_category=...` preflight plus `POST /api/skills/install-upload`: a same-owner custom conflict may be uploaded again with `overwrite=true`, while a different owner's conflict is non-overwritable and returns that owner's username (falling back to user id). The install endpoint repeats the ownership check, including for admins targeting `custom`, so callers cannot bypass the preflight. Public-skill overwrite remains admin-only. - **Installable sentiment analysis package**: `skill-packages/sentiment-analysis.skill` is deliberately separate from the frontend virtual-agent chat page. The chat page goes through Gateway `POST /api/sentiment-agent/stream` (runtime-config URL/Basic Auth in the body, TLS verify off, raw AG-UI SSE passthrough). The skill package's self-contained `config/sentiment-agent.json` owns the external AG-UI address for agent-skill callers; Basic Auth is resolved from `SENTIMENT_AGENT_AUTH_USERNAME` / `SENTIMENT_AGENT_AUTH_PASSWORD` at runtime (not committed into the package). `scripts/sentiment_stream.py` uses only the Python standard library and converts upstream SSE to stable NDJSON events (`thinking_delta`, `answer_delta`, `done`), with a Markdown convenience mode for normal agent calls. Installation/calling guide: `skill-packages/sentiment-analysis/docs/调用与安装说明.md`. - **Chinese name (`name_zh`)**: a Chinese display name stored in the **`skills` DB table** (`SkillRow.name_zh`, `String(128)`, migration `20260609_01`) ? never written to the on-disk SKILL.md, so it works for built-in skills too. Threaded through `SkillStore` create/ensure_legacy/update/update_admin exactly like `detail`; surfaced on `SkillResponse.name_zh` from the ownership record in `_skill_to_response`. Authorization mirrors `detail`: custom skill ? owner or admin; built-in/public ? admin only (row materialized on demand via `ensure_legacy`). Endpoints (in `app/gateway/routers/skills.py`): `PUT /api/skills/{name}/name-zh` (set/clear), `POST /api/skills/{name}/name-zh/generate` (LLM generates one), `POST /api/skills/name-zh/generate-missing` (????: **admin-only** bulk-generate for every skill that lacks one, bounded-concurrency `asyncio.gather`). **Optional model selection**: both generate endpoints accept an optional JSON body `{model_name}` ? an explicit name wins, else the curator model setting, else the main model (`_resolve_skill_name_zh_model_name`). **Model-error surfacing**: `_generate_skill_name_zh` raises `SkillNameZhModelError` when the model itself fails (build/timeout/API error) ? distinct from the model returning an unusable name (? `None`). The single endpoint maps that to `502 ?????,?????????:?`; the bulk endpoint fails fast on a model-build error (502) and otherwise carries a representative `model_error` string in `SkillNameZhBulkResponse` so the UI can say "N ?????????". The frontend (`skill-settings-page.tsx` list + `skill-detail-dialog.tsx` editor, plus the agent skill pickers/badges in `NewAgentPage.tsx` + `agent-card.tsx`) shows `name_zh || name`, offers per-skill "AI ??" plus an **admin-only** header "???????" button, and a compact model dropdown (`components/workspace/model-name-select.tsx`, default = configured model) beside each action. diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index bedd9bdf..c23e9ad1 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -140,6 +140,7 @@ FastAPI application providing REST endpoints for frontend integration: | `GET/PUT /api/mcp/config` | Manage MCP server configurations | | `GET/PUT /api/skills` | List and manage skills | | `POST /api/skills/install` | Install skill from `.skill` archive | +| `POST /api/skills/validate`, `POST /api/skills/install-upload` | Validate and upload `.zip`/`.skill` packages with owner-aware conflicts: owners may explicitly overwrite their own copy; other users' copies report the uploader and cannot be overwritten | | `GET/POST /api/agents` | List visible agents or create a user-owned agent | | `GET/PUT/DELETE /api/agents/{id}` | Read, update, or delete an id-addressed agent | | `GET/POST/PUT/DELETE /api/roundtable-chains` | Manage user-owned roundtable business chains; each seat may optionally carry `position_id` for the separate position-collaboration workspace, plus `human_validation` to mark a Step 2 client-side pause checkpoint after that seat/stage completes (the original roundtable has no extra validation card) | @@ -511,7 +512,10 @@ Key sections: - `sandbox` - Execution environment provider - `skills` - Skills directory paths plus optional prompt routing; `skills.es_query_routing` can inject an editable system-prompt rule that sends flexible Elasticsearch Query DSL - requests to the `es_query` skill, and can be disabled without changing code + requests to the `es_query` skill, and can be disabled without changing code. Each ordinary + Q&A agent build logs `ES query routing prompt registration` with `registered`, eligibility, + character count, and a short SHA-256 fingerprint so operators can verify the exact configured + block reached the final model system prompt without logging its contents - `title` - Auto-title generation settings - `summarization` - Context summarization settings - `subagents` - Subagent system (enabled/disabled) diff --git a/offline-backend-20260512/backend/app/gateway/routers/skills.py b/offline-backend-20260512/backend/app/gateway/routers/skills.py index 07d3f418..7b07264e 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/skills.py +++ b/offline-backend-20260512/backend/app/gateway/routers/skills.py @@ -272,6 +272,14 @@ class SkillUploadValidateResponse(BaseModel): allowed_tools: list[str] = Field(default_factory=list, description="Tools the skill may invoke") conflict: bool = Field(default=False, description="True if a skill with the same name already exists") conflict_kind: str | None = Field(default=None, description="'public' or 'custom' when conflict=true") + can_install_as_is: bool = Field(default=True, description="Whether the archive can be installed without overwrite") + can_overwrite: bool = Field(default=False, description="Whether the caller may overwrite the conflicting skill") + owned_by_me: bool = Field(default=False, description="True when the caller owns the conflicting custom skill") + owned_by_other: bool = Field(default=False, description="True when another user owns the conflicting custom skill") + builtin_conflict: bool = Field(default=False, description="True when the name conflicts with a built-in/public skill") + owner_username: str | None = Field(default=None, description="Friendly username for a conflicting skill owned by another user") + owner_user_id: str | None = Field(default=None, description="Owner user id for a conflicting custom skill") + message: str = Field(default="", description="Human-readable upload guidance") warnings: list[str] = Field(default_factory=list, description="Non-fatal warnings to surface to the user") @@ -972,16 +980,55 @@ async def _persist_uploaded_skill(file: UploadFile) -> Path: "/skills/validate", response_model=SkillUploadValidateResponse, summary="Validate Skill Archive (No Install)", - description="Inspect an uploaded .zip/.skill archive and report parsed metadata + conflicts. Does NOT install or run the LLM security scan.", + description="Inspect an uploaded .zip/.skill archive and report parsed metadata plus ownership-aware conflicts. Does NOT install or run the LLM security scan.", ) async def validate_skill_upload( + request: Request, file: UploadFile = File(..., description="The .zip or .skill archive to validate"), + target_category: SkillCategory = SkillCategory.CUSTOM, config: AppConfig = Depends(get_config), + skill_store: SkillStore = Depends(get_skill_store), ) -> SkillUploadValidateResponse: tmp_path = await _persist_uploaded_skill(file) try: result = get_or_new_skill_storage(app_config=config).validate_skill_archive(tmp_path) - return SkillUploadValidateResponse(**result) + ( + can_install_as_is, + can_overwrite, + owned_by_me, + owned_by_other, + builtin_conflict, + owner_user_id, + owner_username, + message, + ) = await _preview_install_target( + request=request, + skill_store=skill_store, + skill_name=str(result.get("skill_name") or ""), + target_category=target_category, + conflict_kind=result.get("conflict_kind"), + ) + effective_conflict_kind = result.get("conflict_kind") + if effective_conflict_kind is None: + if owned_by_me or owned_by_other: + effective_conflict_kind = "custom" + elif builtin_conflict: + effective_conflict_kind = "public" + return SkillUploadValidateResponse( + **{ + **result, + "conflict": bool(result.get("conflict") or effective_conflict_kind), + "conflict_kind": effective_conflict_kind, + "can_install_as_is": can_install_as_is, + "can_overwrite": can_overwrite, + "owned_by_me": owned_by_me, + "owned_by_other": owned_by_other, + "builtin_conflict": builtin_conflict, + "owner_username": owner_username, + "owner_user_id": owner_user_id, + "message": message, + } + ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except FileNotFoundError as e: @@ -999,7 +1046,7 @@ async def validate_skill_upload( "/skills/install-upload", response_model=SkillInstallResponse, summary="Install Skill from Upload", - description="Install a skill directly from an uploaded .zip/.skill archive. Set ?overwrite=true to replace an existing skill with the same name.", + description="Install a skill directly from an uploaded .zip/.skill archive. Set ?overwrite=true to replace the caller's own custom skill, or an existing public skill when installing as an admin.", ) async def install_skill_upload( request: Request, @@ -1017,22 +1064,30 @@ async def install_skill_upload( tmp_path = await _persist_uploaded_skill(file) try: storage = get_or_new_skill_storage(app_config=config) - # Pre-parse the archive to learn the skill name without writing anything, - # so we can reject the upload before touching the filesystem if another - # user already owns this name. Admins bypass the ownership check since - # they install to the shared public directory. - try: - validation = storage.validate_skill_archive(tmp_path) - preview_name = validation.get("skill_name") if isinstance(validation, dict) else None - except Exception: - preview_name = None - if preview_name and not is_admin: - existing = await skill_store.get_any(preview_name) - if existing and existing.get("owner_user_id") not in (None, user_id): - raise HTTPException( - status_code=409, - detail=f"Skill '{preview_name}' already exists and is owned by another user", - ) + # Re-run the same ownership-aware preflight used by /skills/validate. + # This closes the time-of-check/time-of-use gap and prevents a crafted + # request (including an admin installing into custom/) from replacing a + # different user's skill. + validation = storage.validate_skill_archive(tmp_path) + preview_name = str(validation.get("skill_name") or "") + ( + can_install_as_is, + can_overwrite, + _owned_by_me, + _owned_by_other, + _builtin_conflict, + _owner_user_id, + _owner_username, + conflict_message, + ) = await _preview_install_target( + request=request, + skill_store=skill_store, + skill_name=preview_name, + target_category=target_category, + conflict_kind=validation.get("conflict_kind"), + ) + if (overwrite and not can_install_as_is and not can_overwrite) or (not overwrite and not can_install_as_is): + raise HTTPException(status_code=409, detail=conflict_message) result = await storage.ainstall_skill_from_archive(tmp_path, overwrite=overwrite, require_skill_suffix=False, target_category=target_category) try: from deerflow.skills.usage import mark_uploaded @@ -1075,7 +1130,8 @@ async def _preview_install_target( skill_store: SkillStore, skill_name: str, target_category: SkillCategory, -) -> tuple[bool, bool, bool, bool, str | None, str | None, str]: + conflict_kind: str | None = None, +) -> tuple[bool, bool, bool, bool, bool, str | None, str | None, str]: """Return installability / ownership info for a parsed skill name.""" user_id = _current_user_id(request) @@ -1085,27 +1141,44 @@ async def _preview_install_target( owner_username: str | None = None if owner_user_id: try: - attached = await _attach_owner_usernames( - [SkillResponse(name=skill_name, description="", license=None, category=SkillCategory.CUSTOM, owner_user_id=owner_user_id)] - ) + attached = await _attach_owner_usernames([SkillResponse(name=skill_name, description="", license=None, category=SkillCategory.CUSTOM, owner_user_id=owner_user_id)]) owner_username = attached[0].owner_username if attached else None except Exception: owner_username = None - builtin_conflict = bool(existing and owner_user_id is None) - owned_by_me = bool(existing and owner_user_id == user_id) - owned_by_other = bool(existing and owner_user_id not in (None, user_id)) + custom_conflict = conflict_kind == "custom" or bool(existing and owner_user_id is not None) + builtin_conflict = conflict_kind == "public" or bool(existing and owner_user_id is None and conflict_kind != "custom") + owned_by_me = bool(custom_conflict and owner_user_id == user_id) + owned_by_other = bool(custom_conflict and owner_user_id not in (None, user_id)) + unattributed_custom_conflict = bool(custom_conflict and owner_user_id is None and not builtin_conflict) if target_category == SkillCategory.PUBLIC and not is_admin: - return False, owned_by_me, owned_by_other, builtin_conflict, owner_user_id, owner_username, "Only admin users can install skills to the public directory" + return False, False, owned_by_me, owned_by_other, builtin_conflict, owner_user_id, owner_username, "Only admin users can install skills to the public directory" + + if target_category == SkillCategory.PUBLIC: + if custom_conflict: + who = owner_username or owner_user_id + if owned_by_other and who: + message = f"Skill '{skill_name}' has already been uploaded by user '{who}' and cannot be overwritten." + elif owned_by_me: + message = f"You already own custom skill '{skill_name}'. Switch the target to custom to overwrite it." + else: + message = f"Skill '{skill_name}' already exists as a custom skill and its owner could not be determined." + return False, False, owned_by_me, owned_by_other, builtin_conflict, owner_user_id, owner_username, message + if builtin_conflict: + return False, True, False, False, True, owner_user_id, owner_username, f"Built-in/public skill '{skill_name}' already exists and can be overwritten by an admin." + return True, False, False, False, False, owner_user_id, owner_username, f"Skill '{skill_name}' can be installed." + if builtin_conflict: - return False, owned_by_me, owned_by_other, True, owner_user_id, owner_username, f"Skill '{skill_name}' already exists as a built-in/public skill. Please install it under a different name." + return False, False, owned_by_me, owned_by_other, True, owner_user_id, owner_username, f"Skill '{skill_name}' already exists as a built-in/public skill. Please install it under a different name." if owned_by_other: - who = owner_username or "another user" - return False, owned_by_me, True, builtin_conflict, owner_user_id, owner_username, f"Skill '{skill_name}' has already been installed by {who}. Please choose a different name." + who = owner_username or owner_user_id or "another user" + return False, False, False, True, False, owner_user_id, owner_username, f"Skill '{skill_name}' has already been uploaded by user '{who}' and cannot be overwritten." if owned_by_me: - return False, True, False, builtin_conflict, owner_user_id, owner_username, f"You have already installed skill '{skill_name}'. You can keep using it or install this copy under a different name." - return True, False, False, builtin_conflict, owner_user_id, owner_username, f"Skill '{skill_name}' can be installed." + return False, True, True, False, False, owner_user_id, owner_username, f"You have already uploaded skill '{skill_name}'. You may upload it again and overwrite your existing copy." + if unattributed_custom_conflict: + return False, False, False, False, False, owner_user_id, owner_username, f"Skill '{skill_name}' already exists, but its owner could not be determined. It cannot be overwritten safely." + return True, False, False, False, False, owner_user_id, owner_username, f"Skill '{skill_name}' can be installed." async def _build_renamed_skill_archive( @@ -1193,11 +1266,12 @@ async def preview_skill_install( skill_file_path = await aresolve_thread_virtual_path(body.thread_id, body.path) validation = get_or_new_skill_storage(app_config=config).validate_skill_archive(skill_file_path, require_skill_suffix=False) skill_name = str(validation.get("skill_name") or "") - can_install_as_is, owned_by_me, owned_by_other, builtin_conflict, owner_user_id, owner_username, message = await _preview_install_target( + can_install_as_is, _can_overwrite, owned_by_me, owned_by_other, builtin_conflict, owner_user_id, owner_username, message = await _preview_install_target( request=request, skill_store=skill_store, skill_name=skill_name, target_category=body.target_category, + conflict_kind=validation.get("conflict_kind"), ) return SkillInstallPreviewResponse( ok=True, @@ -1248,11 +1322,12 @@ async def install_skill( storage = get_or_new_skill_storage(app_config=config) validation = storage.validate_skill_archive(skill_file_path, require_skill_suffix=False) preview_name = requested_name or str(validation.get("skill_name") or "") - can_install_as_is, owned_by_me, owned_by_other, builtin_conflict, _owner_user_id, _owner_username, message = await _preview_install_target( + can_install_as_is, _can_overwrite, owned_by_me, owned_by_other, builtin_conflict, _owner_user_id, _owner_username, message = await _preview_install_target( request=request, skill_store=skill_store, skill_name=preview_name, target_category=target_category, + conflict_kind=(validation.get("conflict_kind") if requested_name is None or requested_name == validation.get("skill_name") else None), ) if not can_install_as_is: if requested_name and requested_name != validation.get("skill_name") and not owned_by_other and not builtin_conflict: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py index 733110f1..f27ae6d2 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -1,4 +1,5 @@ -import logging +import hashlib +import logging from langchain.agents import create_agent from langchain.agents.middleware import AgentMiddleware @@ -37,7 +38,45 @@ logger = logging.getLogger(__name__) # 写作配置 Q&A 现在「先生成大纲并与用户多轮确认 → 再提交配置」，需要更多来回，故上限放宽。 # 仍是兜底安全阀：到顶才用 tool_choice 强制提交，正常流程由用户确认大纲后自然提交。 WRITING_SETUP_MAX_ROUNDS = 8 -REPORT_STRUCTURE_SETUP_MAX_ROUNDS = 8 +REPORT_STRUCTURE_SETUP_MAX_ROUNDS = 8 + +_ES_QUERY_ROUTING_OPEN_TAG = '' + + +def _log_es_query_routing_registration( + *, + system_prompt: str, + app_config: AppConfig, + agent_id: str | None, + available_skills: set[str] | None, +) -> None: + """Log whether the exact configured ES routing block reached this Q&A prompt.""" + routing = app_config.skills.es_query_routing + configured_prompt = (routing.prompt or "").strip() + expected_block = ( + f"{_ES_QUERY_ROUTING_OPEN_TAG}\n{configured_prompt}\n" + if configured_prompt + else "" + ) + eligible = available_skills is None or "es_query" in available_skills + registered = bool(expected_block) and expected_block in system_prompt + fingerprint = ( + hashlib.sha256(configured_prompt.encode("utf-8")).hexdigest()[:12] + if configured_prompt + else "-" + ) + + log = logger.info if registered or not routing.enabled or not eligible else logger.warning + log( + "ES query routing prompt registration: agent_id=%s enabled=%s eligible=%s " + "registered=%s prompt_chars=%d prompt_sha256=%s", + agent_id or "default", + routing.enabled, + eligible, + registered, + len(configured_prompt), + fingerprint, + ) def _get_runtime_config(config: RunnableConfig) -> dict: @@ -675,8 +714,8 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig): writing_mode=writing_mode, writing_artifact_path=writing_artifact_path, ) - else: - system_prompt = apply_prompt_template( + else: + system_prompt = apply_prompt_template( subagent_enabled=subagent_enabled, max_concurrent_subagents=max_concurrent_subagents, agent_id=agent_id, @@ -691,8 +730,15 @@ def _make_lead_agent(config: RunnableConfig, *, app_config: AppConfig): notebook_source_ids=notebook_source_ids, is_scheduled_run=bool(cfg.get("is_scheduled_run", False)), ) - - return create_agent( + + _log_es_query_routing_registration( + system_prompt=system_prompt, + app_config=resolved_app_config, + agent_id=agent_id, + available_skills=prompt_available_skills, + ) + + return create_agent( model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, force_disable_thinking=thinking_force_disabled, app_config=resolved_app_config), tools=default_tools, middleware=_build_middlewares(config, model_name=model_name, agent_name=agent_id, app_config=resolved_app_config), diff --git a/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py index ec6080b4..e11728ae 100644 --- a/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py +++ b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py @@ -1,5 +1,7 @@ +import logging from types import SimpleNamespace +from deerflow.agents.lead_agent import agent as agent_module from deerflow.agents.lead_agent import prompt as prompt_module from deerflow.config.skills_config import EsQueryRoutingConfig, SkillsConfig @@ -70,6 +72,44 @@ def test_es_query_routing_is_injected_into_lead_and_allowed_custom_agent_prompts assert '' not in denied_custom_prompt +def test_es_query_routing_registration_log_confirms_exact_final_prompt(caplog) -> None: + config = _app_config(enabled=True, prompt="自定义 ES 路由规则") + system_prompt = '\n自定义 ES 路由规则\n' + + with caplog.at_level(logging.INFO, logger=agent_module.__name__): + agent_module._log_es_query_routing_registration( + system_prompt=system_prompt, + app_config=config, + agent_id=None, + available_skills=None, + ) + + assert "ES query routing prompt registration" in caplog.text + assert "agent_id=default" in caplog.text + assert "enabled=True" in caplog.text + assert "eligible=True" in caplog.text + assert "registered=True" in caplog.text + assert "prompt_chars=11" in caplog.text + assert "prompt_sha256=" in caplog.text + + +def test_es_query_routing_registration_log_explains_custom_agent_exclusion(caplog) -> None: + config = _app_config(enabled=True, prompt="自定义 ES 路由规则") + + with caplog.at_level(logging.INFO, logger=agent_module.__name__): + agent_module._log_es_query_routing_registration( + system_prompt="custom agent prompt", + app_config=config, + agent_id="agent-without-es", + available_skills={"other_skill"}, + ) + + assert "agent_id=agent-without-es" in caplog.text + assert "enabled=True" in caplog.text + assert "eligible=False" in caplog.text + assert "registered=False" in caplog.text + + def test_formal_markdown_rules_are_scoped_to_ordinary_qa(monkeypatch) -> None: config = _app_config(enabled=False) monkeypatch.setattr(prompt_module, "_get_memory_context", lambda *args, **kwargs: "") diff --git a/offline-backend-20260512/backend/tests/test_skills_install_upload.py b/offline-backend-20260512/backend/tests/test_skills_install_upload.py index 969eea15..975cef69 100644 --- a/offline-backend-20260512/backend/tests/test_skills_install_upload.py +++ b/offline-backend-20260512/backend/tests/test_skills_install_upload.py @@ -5,11 +5,16 @@ from __future__ import annotations import io import zipfile from pathlib import Path +from types import SimpleNamespace import pytest +from fastapi import HTTPException +from app.gateway.routers import skills as skills_router +from deerflow.persistence.skills import MemorySkillStore from deerflow.skills.installer import SkillAlreadyExistsError from deerflow.skills.storage.local_skill_storage import LocalSkillStorage +from deerflow.skills.types import SkillCategory def _make_skill_zip( @@ -239,3 +244,218 @@ async def test_install_refuses_to_overwrite_public_skill( await storage.ainstall_skill_from_archive( archive, overwrite=True, require_skill_suffix=False ) + + +# ---------- ownership-aware direct upload conflicts ---------- + + +def _request(user_id: str, *, is_admin: bool = False) -> SimpleNamespace: + return SimpleNamespace( + state=SimpleNamespace( + user=SimpleNamespace( + id=user_id, + system_role="admin" if is_admin else "user", + ) + ) + ) + + +async def _attach_test_owner_names(responses): + for response in responses: + if response.owner_user_id == "user-b": + response.owner_username = "bob" + return responses + + +class _UploadStorage: + def __init__(self) -> None: + self.install_calls = 0 + + def validate_skill_archive(self, _path: Path) -> dict: + return { + "ok": True, + "skill_name": "demo-skill", + "description": "demo", + "license": None, + "allowed_tools": [], + "conflict": True, + "conflict_kind": "custom", + "warnings": [], + } + + async def ainstall_skill_from_archive(self, *_args, **_kwargs) -> dict: + self.install_calls += 1 + return { + "success": True, + "skill_name": "demo-skill", + "message": "installed", + "overwritten": True, + "category": "custom", + } + + +@pytest.mark.asyncio +async def test_upload_preflight_allows_owner_to_overwrite( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = MemorySkillStore() + await store.create({"name": "demo-skill", "owner_user_id": "user-a"}) + monkeypatch.setattr(skills_router, "_attach_owner_usernames", _attach_test_owner_names) + + result = await skills_router._preview_install_target( + request=_request("user-a"), + skill_store=store, + skill_name="demo-skill", + target_category=SkillCategory.CUSTOM, + conflict_kind="custom", + ) + + can_install_as_is, can_overwrite, owned_by_me, owned_by_other, *_rest = result + assert can_install_as_is is False + assert can_overwrite is True + assert owned_by_me is True + assert owned_by_other is False + assert "upload it again and overwrite" in result[-1] + + +@pytest.mark.asyncio +async def test_upload_preflight_names_the_other_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = MemorySkillStore() + await store.create({"name": "demo-skill", "owner_user_id": "user-b"}) + monkeypatch.setattr(skills_router, "_attach_owner_usernames", _attach_test_owner_names) + + result = await skills_router._preview_install_target( + request=_request("user-a"), + skill_store=store, + skill_name="demo-skill", + target_category=SkillCategory.CUSTOM, + conflict_kind="custom", + ) + + can_install_as_is, can_overwrite, owned_by_me, owned_by_other = result[:4] + assert can_install_as_is is False + assert can_overwrite is False + assert owned_by_me is False + assert owned_by_other is True + assert result[6] == "bob" + assert "user 'bob'" in result[-1] + + +@pytest.mark.asyncio +async def test_validate_upload_surfaces_the_other_owner( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + store = MemorySkillStore() + await store.create({"name": "demo-skill", "owner_user_id": "user-b"}) + storage = _UploadStorage() + upload_path = tmp_path / "validate.skill" + upload_path.write_bytes(b"placeholder") + + async def _persist(_file) -> Path: + return upload_path + + monkeypatch.setattr(skills_router, "_persist_uploaded_skill", _persist) + monkeypatch.setattr( + skills_router, + "get_or_new_skill_storage", + lambda **_kwargs: storage, + ) + monkeypatch.setattr( + skills_router, + "_attach_owner_usernames", + _attach_test_owner_names, + ) + + response = await skills_router.validate_skill_upload( + request=_request("user-a"), + file=object(), + target_category=SkillCategory.CUSTOM, + config=object(), + skill_store=store, + ) + + assert response.conflict is True + assert response.conflict_kind == "custom" + assert response.can_overwrite is False + assert response.owned_by_other is True + assert response.owner_username == "bob" + + +@pytest.mark.asyncio +async def test_direct_upload_owner_can_overwrite_own_skill( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + store = MemorySkillStore() + await store.create({"name": "demo-skill", "owner_user_id": "user-a"}) + storage = _UploadStorage() + upload_path = tmp_path / "owner-upload.skill" + upload_path.write_bytes(b"placeholder") + + async def _persist(_file) -> Path: + return upload_path + + async def _refresh() -> None: + return None + + monkeypatch.setattr(skills_router, "_persist_uploaded_skill", _persist) + monkeypatch.setattr( + skills_router, + "get_or_new_skill_storage", + lambda **_kwargs: storage, + ) + monkeypatch.setattr( + skills_router, + "refresh_skills_system_prompt_cache_async", + _refresh, + ) + monkeypatch.setattr("deerflow.skills.usage.mark_uploaded", lambda _name: None) + + response = await skills_router.install_skill_upload( + request=_request("user-a"), + file=object(), + overwrite=True, + target_category=SkillCategory.CUSTOM, + config=object(), + skill_store=store, + ) + + assert response.success is True + assert response.overwritten is True + assert storage.install_calls == 1 + + +@pytest.mark.asyncio +async def test_direct_upload_rechecks_owner_before_overwrite( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + store = MemorySkillStore() + await store.create({"name": "demo-skill", "owner_user_id": "user-b"}) + storage = _UploadStorage() + upload_path = tmp_path / "upload.skill" + upload_path.write_bytes(b"placeholder") + + async def _persist(_file) -> Path: + return upload_path + + monkeypatch.setattr(skills_router, "_persist_uploaded_skill", _persist) + monkeypatch.setattr(skills_router, "get_or_new_skill_storage", lambda **_kwargs: storage) + monkeypatch.setattr(skills_router, "_attach_owner_usernames", _attach_test_owner_names) + + with pytest.raises(HTTPException) as exc_info: + await skills_router.install_skill_upload( + request=_request("user-a", is_admin=True), + file=object(), + overwrite=True, + target_category=SkillCategory.CUSTOM, + config=object(), + skill_store=store, + ) + + assert exc_info.value.status_code == 409 + assert "user 'bob'" in str(exc_info.value.detail) + assert storage.install_calls == 0