From 5f6ab41798461ee2860b0612761fdd4cbe6b6a5a Mon Sep 17 00:00:00 2001 From: ZyGuo Date: Fri, 5 Jun 2026 14:15:42 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20iframe=E4=BC=A0=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=BF=9B=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pages/iframe-chat-submit.test.ts | 34 +++++++ frontend-web/src/pages/iframe-chat-submit.ts | 49 +++++++++- .../agents/middlewares/uploads_middleware.py | 96 +++++++++++++++---- 3 files changed, 158 insertions(+), 21 deletions(-) diff --git a/frontend-web/src/pages/iframe-chat-submit.test.ts b/frontend-web/src/pages/iframe-chat-submit.test.ts index df236068..ac0514b7 100644 --- a/frontend-web/src/pages/iframe-chat-submit.test.ts +++ b/frontend-web/src/pages/iframe-chat-submit.test.ts @@ -100,3 +100,37 @@ test("maps iframe model selection into stream context model_name", () => { model_name: "OpenAI3", }); }); + +test("maps externally uploaded files into stream additional_kwargs.files", () => { + const result = parseBrainyChatSubmitMessage({ + source: "brainy-extension", + type: "brainy:chat-submit", + requestId: "iframe-789", + payload: { + text: "网摘", + prompt: "网摘", + uploadedFiles: [ + { + filename: "LLM Wiki 接口说明.md", + size: "7002", + path: "/mnt/user-data/uploads/LLM Wiki 接口说明.md", + status: "uploaded", + artifact_url: + "/api/threads/445ca7a7-9737-44f0-8eea-6887776f4f9c/artifacts/mnt/user-data/uploads/LLM%20Wiki%20%E6%8E%A5%E5%8F%A3%E8%AF%B4%E6%98%8E.md", + }, + ], + }, + }); + + assert.equal(result.submitText, "网摘"); + assert.deepEqual(result.additionalKwargs?.files, [ + { + filename: "LLM Wiki 接口说明.md", + size: "7002", + path: "/mnt/user-data/uploads/LLM Wiki 接口说明.md", + status: "uploaded", + artifact_url: + "/api/threads/445ca7a7-9737-44f0-8eea-6887776f4f9c/artifacts/mnt/user-data/uploads/LLM%20Wiki%20%E6%8E%A5%E5%8F%A3%E8%AF%B4%E6%98%8E.md", + }, + ]); +}); diff --git a/frontend-web/src/pages/iframe-chat-submit.ts b/frontend-web/src/pages/iframe-chat-submit.ts index 094e05f6..45cf9d0d 100644 --- a/frontend-web/src/pages/iframe-chat-submit.ts +++ b/frontend-web/src/pages/iframe-chat-submit.ts @@ -26,6 +26,14 @@ type BrainyModelSelection = { value?: string; }; +type BrainyUploadedFile = { + filename?: string; + size?: string | number; + path?: string; + status?: string; + artifact_url?: string; +}; + export type BrainyChatSubmitMessage = { source?: string; type?: string; @@ -35,6 +43,7 @@ export type BrainyChatSubmitMessage = { pageContext?: BrainyPageContext; screenshot?: BrainyScreenshot; model?: BrainyModelSelection; + uploadedFiles?: BrainyUploadedFile[]; }; requestId?: string; }; @@ -253,6 +262,36 @@ function buildScreenshotFileParts( return []; } +function normalizeUploadedFiles( + uploadedFiles: BrainyUploadedFile[] | undefined, +): Record[] { + if (!Array.isArray(uploadedFiles)) { + return []; + } + + return uploadedFiles + .map((file) => { + const filename = trimToUndefined(file.filename); + if (!filename) { + return null; + } + + const size = String(file.size ?? "0"); + const path = trimToUndefined(file.path); + const status = trimToUndefined(file.status); + const artifactUrl = trimToUndefined(file.artifact_url); + + return { + filename, + size, + ...(path ? { path } : {}), + ...(status ? { status } : {}), + ...(artifactUrl ? { artifact_url: artifactUrl } : {}), + }; + }) + .filter((file): file is Record => file !== null); +} + export function parseBrainyChatSubmitMessage( data: BrainyChatSubmitMessage, ): ParsedBrainyChatSubmitMessage { @@ -268,6 +307,7 @@ export function parseBrainyChatSubmitMessage( }); const requestId = trimToUndefined(data.requestId); const modelName = trimToUndefined(data.payload?.model?.value); + const uploadedFiles = normalizeUploadedFiles(data.payload?.uploadedFiles); const iframeContext: Record = {}; if (requestId) { @@ -281,6 +321,13 @@ export function parseBrainyChatSubmitMessage( submitText, files, ...(Object.keys(iframeContext).length > 0 ? { iframeContext } : {}), - ...(promptPrefix ? { additionalKwargs: { prompt_prefix: promptPrefix } } : {}), + ...(promptPrefix || uploadedFiles.length > 0 + ? { + additionalKwargs: { + ...(promptPrefix ? { prompt_prefix: promptPrefix } : {}), + ...(uploadedFiles.length > 0 ? { files: uploadedFiles } : {}), + }, + } + : {}), }; } diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py index 43c93b69..e6fa4a2c 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/middlewares/uploads_middleware.py @@ -1,8 +1,10 @@ """Middleware to inject uploaded files information into agent context.""" -import logging -from pathlib import Path -from typing import NotRequired, override +import logging +import shutil +from pathlib import Path +from typing import NotRequired, override +from urllib.parse import unquote from langchain.agents import AgentState from langchain.agents.middleware import AgentMiddleware @@ -13,7 +15,7 @@ from deerflow.config.paths import Paths, get_paths from deerflow.runtime.user_context import get_effective_user_id from deerflow.utils.file_conversion import extract_outline -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) _OUTLINE_PREVIEW_LINES = 5 @@ -22,7 +24,8 @@ _OUTLINE_PREVIEW_LINES = 5 # configured vision model), never by reading their bytes. Listing them with the # generic read_file/grep guidance made the agent try to parse image binaries # with bash/python, which fails and wastes turns. -_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"} +_IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp"} +_ARTIFACT_URL_MARKER = "/artifacts/" def _extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]: @@ -69,7 +72,7 @@ class UploadsMiddlewareState(AgentState): uploaded_files: NotRequired[list[dict] | None] -class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): +class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): """Middleware to inject uploaded files information into the agent context. Reads file metadata from the current message's additional_kwargs.files @@ -122,7 +125,7 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): lines.append(" Use `grep` to search for keywords (e.g. `grep(pattern='keyword', path='/mnt/user-data/uploads/')`).") lines.append("") - def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str: + def _create_files_message(self, new_files: list[dict], historical_files: list[dict]) -> str: """Create a formatted message listing uploaded files. Args: @@ -160,9 +163,61 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): lines.append("- Only fall back to web search if the file content is clearly insufficient to answer the question.") lines.append("") - return "\n".join(lines) - - def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None: + return "\n".join(lines) + + def _import_file_from_artifact_url( + self, + file_info: dict, + uploads_dir: Path | None, + ) -> bool: + """Copy a previously uploaded file into the current thread when possible. + + iframe-style callers may pass file metadata that points at an upload + from another DeerFlow thread. The current run thread still needs a real + file under its own ``/mnt/user-data/uploads`` directory, otherwise this + middleware will filter the entry out during existence checks. + """ + filename = file_info.get("filename") or "" + artifact_url = str(file_info.get("artifact_url") or "").strip() + if not filename or uploads_dir is None or not artifact_url: + return False + + if not artifact_url.startswith("/api/threads/") or _ARTIFACT_URL_MARKER not in artifact_url: + return False + + prefix, encoded_virtual_path = artifact_url.split(_ARTIFACT_URL_MARKER, 1) + source_thread_id = prefix.removeprefix("/api/threads/").strip("/") + if not source_thread_id or not encoded_virtual_path: + return False + + virtual_path = "/" + unquote(encoded_virtual_path.lstrip("/")) + if Path(virtual_path).name != filename: + return False + + try: + source_path = self._paths.resolve_virtual_path( + source_thread_id, + virtual_path, + user_id=get_effective_user_id(), + ) + except ValueError: + logger.debug("Failed to resolve uploaded artifact path: %s", artifact_url, exc_info=True) + return False + + if not source_path.is_file(): + return False + + uploads_dir.mkdir(parents=True, exist_ok=True) + target_path = uploads_dir / filename + shutil.copy2(source_path, target_path) + + source_markdown = source_path.with_suffix(".md") + if source_markdown.is_file() and source_markdown != source_path: + shutil.copy2(source_markdown, target_path.with_suffix(".md")) + + return True + + def _files_from_kwargs(self, message: HumanMessage, uploads_dir: Path | None = None) -> list[dict] | None: """Extract file info from message additional_kwargs.files. The frontend sends uploaded file metadata in additional_kwargs.files @@ -183,16 +238,17 @@ class UploadsMiddleware(AgentMiddleware[UploadsMiddlewareState]): files = [] for f in kwargs_files: - if not isinstance(f, dict): - continue - filename = f.get("filename") or "" - if not filename or Path(filename).name != filename: - continue - if uploads_dir is not None and not (uploads_dir / filename).is_file(): - continue - files.append( - { - "filename": filename, + if not isinstance(f, dict): + continue + filename = f.get("filename") or "" + if not filename or Path(filename).name != filename: + continue + if uploads_dir is not None and not (uploads_dir / filename).is_file(): + if not self._import_file_from_artifact_url(f, uploads_dir): + continue + files.append( + { + "filename": filename, "size": int(f.get("size") or 0), "path": f"/mnt/user-data/uploads/{filename}", "extension": Path(filename).suffix,