From 5aa1fdca04150f23afba43b37db05ab847cc2dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Tue, 2 Jun 2026 16:43:10 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=86=E5=9B=BE=E4=B8=8E=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0/=E9=A2=84=E8=A7=88:=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E8=A7=86=E8=A7=89=E6=A8=A1=E5=9E=8B=20+=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E4=B8=8A=E4=BC=A0=E4=B8=8E=E5=9B=9E=E6=98=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 后端: - 新增可配置识图模型(config: vision.model_name/prompt/max_tokens)。配置后 view_image 委托该模型识别并返回文字结果,文本主模型也能“看图”;未配置时回退到主模型自带视觉能力。新增 deerflow/models/vision.py 与 VisionConfig。 - 修复 Windows 上传全部报 unsafe:open_upload_file_no_symlink 不再强制要求 POSIX 专有的 O_NOFOLLOW,缺失时降级(仍用 lstat/fstat 防符号链接),Windows 追加 O_BINARY。 - 修复 normalize_input 丢弃 human 消息 additional_kwargs:保留 files/client_ts 等,使上传文件能回传到 UI 并被 UploadsMiddleware 识别。 - 约束 agent 行为:上传图片时引导只用 view_image,禁止用 read_file/grep/bash/python 解析图片字节;view_image 工具说明同步加固。 - 启用文档转换(uploads.auto_convert_documents),PDF/Word/Excel/PPT 自动转 Markdown 供 read_file/grep 读取。 - 订正若干模型错误的 supports_vision 标记。 前端: - 修复上传文件不回显:parseUploadedFiles 不再因 "(empty)" 提前返回。 - 修复 artifact 401 破图:改用带 Authorization 头的 apiFetch 取 blob 显示(新增 useAuthedObjectUrl)。 - 新增上传文件预览弹窗:点击卡片可查看图片/文本/Markdown、Office(读转换后的 .md)、PDF;文件卡片改为可点击。 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../workspace/messages/message-list-item.tsx | 117 +++++--- .../messages/uploaded-file-preview-dialog.tsx | 278 ++++++++++++++++++ frontend-web/src/core/messages/utils.ts | 8 +- offline-backend-20260512/backend/CLAUDE.md | 19 +- .../backend/app/gateway/services.py | 22 +- offline-backend-20260512/backend/config.yaml | 37 ++- .../deerflow/agents/lead_agent/agent.py | 12 +- .../agents/middlewares/uploads_middleware.py | 16 + .../middlewares/view_image_middleware.py | 6 + .../harness/deerflow/config/app_config.py | 2 + .../harness/deerflow/config/vision_config.py | 29 ++ .../harness/deerflow/models/__init__.py | 9 +- .../harness/deerflow/models/vision.py | 121 ++++++++ .../tools/builtins/view_image_tool.py | 42 ++- .../packages/harness/deerflow/tools/tools.py | 11 +- .../harness/deerflow/uploads/manager.py | 17 +- offline-backend-20260512/config.yaml | 34 ++- 17 files changed, 706 insertions(+), 74 deletions(-) create mode 100644 frontend-web/src/components/workspace/messages/uploaded-file-preview-dialog.tsx create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/config/vision_config.py create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/models/vision.py diff --git a/frontend-web/src/components/workspace/messages/message-list-item.tsx b/frontend-web/src/components/workspace/messages/message-list-item.tsx index 4daa04ce..70e42054 100644 --- a/frontend-web/src/components/workspace/messages/message-list-item.tsx +++ b/frontend-web/src/components/workspace/messages/message-list-item.tsx @@ -5,6 +5,7 @@ import { useCallback, useContext, useMemo, + useState, type AnchorHTMLAttributes, type ImgHTMLAttributes, } from "react"; @@ -24,6 +25,10 @@ import { } from "@/components/ai-elements/reasoning"; import { Task, TaskTrigger } from "@/components/ai-elements/task"; import { Badge } from "@/components/ui/badge"; +import { + UploadedFilePreviewDialog, + useAuthedObjectUrl, +} from "@/components/workspace/messages/uploaded-file-preview-dialog"; import { resolveArtifactURL } from "@/core/artifacts/utils"; import { useChatAskTimes } from "@/core/chat-timestamps"; import { getBackendBaseURL } from "@/core/config"; @@ -466,8 +471,17 @@ function RichFileCard({ }) { const { t } = useI18n(); const resolveArtifact = useArtifactResolver(threadId); + const [previewOpen, setPreviewOpen] = useState(false); const isUploading = file.status === "uploading"; const isImage = isImageFile(file.filename); + const fileUrl = file.path ? resolveArtifact(file.path) : null; + // Thumbnails must be fetched with auth: the artifact route is gated and a + // plain sends no Authorization header → 401 (broken image). Fetch + // the bytes via apiFetch and display the resulting object URL instead. + const { objectUrl: thumbUrl, loading: thumbLoading } = useAuthedObjectUrl( + fileUrl, + isImage && !isUploading && !!fileUrl, + ); if (isUploading) { return ( @@ -496,50 +510,81 @@ function RichFileCard({ ); } - if (!file.path) return null; + if (!fileUrl) return null; - const fileUrl = resolveArtifact(file.path); + const preview = ( + + ); if (isImage) { return ( - - {file.filename} - + <> + + {preview} + ); } return ( -
-
- - - {file.filename} - -
-
- - {getFileTypeLabel(file.filename)} - - - {formatBytes(file.size)} - -
-
+ <> + + {preview} + ); } diff --git a/frontend-web/src/components/workspace/messages/uploaded-file-preview-dialog.tsx b/frontend-web/src/components/workspace/messages/uploaded-file-preview-dialog.tsx new file mode 100644 index 00000000..0141242e --- /dev/null +++ b/frontend-web/src/components/workspace/messages/uploaded-file-preview-dialog.tsx @@ -0,0 +1,278 @@ +"use client"; + +import { DownloadIcon, Loader2 } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { MarkdownContent } from "@/components/workspace/messages/markdown-content"; +import { apiFetch } from "@/core/api/fetch-client"; +import { useI18n } from "@/core/i18n/hooks"; +import { reasoningPlugins } from "@/core/streamdown"; + +const markdownClassName = + "!h-auto !w-full max-w-none min-w-0 text-sm leading-7 " + + "[&_h1]:mt-0 [&_h1]:mb-3 [&_h1]:text-xl [&_h1]:font-semibold " + + "[&_h2]:mt-6 [&_h2]:mb-2 [&_h2]:border-b [&_h2]:pb-2 [&_h2]:text-lg [&_h2]:font-semibold " + + "[&_h3]:mt-5 [&_h3]:mb-2 [&_h3]:text-base [&_h3]:font-semibold " + + "[&_p]:my-3 [&_ul]:my-3 [&_ol]:my-3 [&_li]:my-1.5 " + + "[&_ul]:list-disc [&_ol]:list-decimal [&_ul]:pl-5 [&_ol]:pl-5 " + + "[&_table]:my-4 [&_table]:w-full [&_table]:border-collapse " + + "[&_th]:border [&_th]:px-2 [&_th]:py-1 [&_td]:border [&_td]:px-2 [&_td]:py-1 " + + "[&_blockquote]:my-4 [&_blockquote]:border-l-4 [&_blockquote]:pl-4 [&_blockquote]:text-muted-foreground " + + "[&_code]:rounded [&_code]:border [&_code]:bg-muted [&_code]:px-1.5 [&_code]:py-0.5 [&_code]:font-mono [&_code]:text-xs " + + "[&_pre]:my-4 [&_pre]:max-w-full [&_pre]:overflow-x-auto [&_pre]:rounded-lg [&_pre]:border [&_pre]:bg-muted [&_pre]:p-4 " + + "[&_pre_code]:border-0 [&_pre_code]:bg-transparent [&_pre_code]:p-0"; + +const IMAGE_EXT = ["png", "jpg", "jpeg", "gif", "webp", "svg", "bmp"]; +const TEXT_EXT = [ + "txt", "md", "markdown", "csv", "tsv", "json", "log", "yaml", "yml", + "xml", "html", "htm", "css", "js", "ts", "tsx", "jsx", "py", "java", + "go", "rs", "c", "cpp", "h", "hpp", "sh", "sql", "ini", "toml", "env", +]; +// Office documents are converted to a sibling Markdown file server-side +// (uploads.auto_convert_documents). We preview that .md instead of the binary. +const OFFICE_EXT = ["xlsx", "xls", "docx", "doc", "pptx", "ppt"]; + +function extOf(filename: string): string { + const i = filename.lastIndexOf("."); + return i >= 0 ? filename.slice(i + 1).toLowerCase() : ""; +} + +type PreviewKind = "image" | "text" | "office" | "pdf" | "binary"; + +function kindOf(filename: string): PreviewKind { + const ext = extOf(filename); + if (IMAGE_EXT.includes(ext)) return "image"; + if (ext === "pdf") return "pdf"; + if (OFFICE_EXT.includes(ext)) return "office"; + if (TEXT_EXT.includes(ext)) return "text"; + return "binary"; +} + +function withReplacedExtension(url: string): string { + // Replace the path extension with .md, keeping any query string intact. + const [base, query] = url.split("?"); + const i = base.lastIndexOf("."); + const swapped = (i >= 0 ? base.slice(0, i) : base) + ".md"; + return query ? `${swapped}?${query}` : swapped; +} + +/** + * Fetch an auth-gated artifact (the artifact route requires an + * `Authorization: Bearer` header, which a plain /