diff --git a/frontend-web/src/pages/IframeChatPage.tsx b/frontend-web/src/pages/IframeChatPage.tsx index 79f37082..821b0ef5 100644 --- a/frontend-web/src/pages/IframeChatPage.tsx +++ b/frontend-web/src/pages/IframeChatPage.tsx @@ -30,21 +30,10 @@ import { useThreadStream } from "@/core/threads/hooks"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { cn } from "@/lib/utils"; - -type BrainyChatSubmitMessage = { - source?: string; - type?: string; - payload?: { - text?: string; - prompt?: string; - pageContext?: { - title?: string; - content?: string; - url?: string; - }; - }; - requestId?: string; -}; +import { + type BrainyChatSubmitMessage, + parseBrainyChatSubmitMessage, +} from "./iframe-chat-submit"; type BrainyChatStatus = "started" | "finished"; @@ -62,47 +51,6 @@ function postBrainyChatStatus(status: BrainyChatStatus, requestId: string) { ); } -function buildIframeMessageText({ - text, - prompt, - pageContext, -}: { - text: string; - prompt: string; - pageContext?: - | { - title?: string; - content?: string; - url?: string; - } - | undefined; -}) { - const primaryText = text || prompt; - const sections = [primaryText]; - - if (prompt && prompt !== primaryText) { - sections.push(`\n${prompt}\n`); - } - - if (pageContext) { - const pageContextParts = [ - pageContext.title ? `\n${pageContext.title}\n` : null, - pageContext.url ? `\n${pageContext.url}\n` : null, - pageContext.content - ? `\n${pageContext.content}\n` - : null, - ].filter(Boolean); - - if (pageContextParts.length > 0) { - sections.push( - `\n${pageContextParts.join("\n\n")}\n`, - ); - } - } - - return sections.join("\n\n"); -} - export default function IframeChatPage() { const routeBase = useRouteBase(); const router = useRouter(); @@ -265,44 +213,23 @@ export default function IframeChatPage() { if (data?.source !== "brainy-extension") return; if (data?.type !== "brainy:chat-submit") return; - const text = data.payload?.text?.trim() ?? ""; - const prompt = data.payload?.prompt?.trim() ?? ""; - const rawPageContext = data.payload?.pageContext; - const pageContext = rawPageContext - ? { - ...(rawPageContext.title?.trim() - ? { title: rawPageContext.title.trim() } - : {}), - ...(rawPageContext.content?.trim() - ? { content: rawPageContext.content.trim() } - : {}), - ...(rawPageContext.url?.trim() - ? { url: rawPageContext.url.trim() } - : {}), - } - : undefined; - const submitText = buildIframeMessageText({ text, prompt, pageContext }); + const { submitText, files, iframeContext } = + parseBrainyChatSubmitMessage(data); console.log("收到插件消息", { - text, - prompt, - pageContext, + text: data.payload?.text, + prompt: data.payload?.prompt, + pageContext: data.payload?.pageContext, + screenshot: data.payload?.screenshot, requestId: data.requestId, + filesCount: files.length, }); - if (!submitText) return; + if (!submitText && files.length === 0) return; pendingRequestIdRef.current = data.requestId ?? null; - const iframeContext: Record = {}; - if (data.requestId) { - iframeContext.brainy_request_id = data.requestId; - } - - handleSubmit( - { text: submitText, files: [] }, - Object.keys(iframeContext).length > 0 ? iframeContext : undefined, - ); + handleSubmit({ text: submitText, files }, iframeContext); }; window.addEventListener("message", handleMessage); diff --git a/frontend-web/src/pages/iframe-chat-submit.test.ts b/frontend-web/src/pages/iframe-chat-submit.test.ts new file mode 100644 index 00000000..690570fd --- /dev/null +++ b/frontend-web/src/pages/iframe-chat-submit.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parseBrainyChatSubmitMessage } from "./iframe-chat-submit"; + +test("parses iframe text, page context, and screenshot into a prompt message", () => { + const result = parseBrainyChatSubmitMessage({ + source: "brainy-extension", + type: "brainy:chat-submit", + requestId: "iframe-123", + payload: { + text: "提取并翻译", + prompt: "提取图片中的文字并翻译成简体中文", + pageContext: { + title: "UN News", + content: "Banning children from social media is not the answer.", + url: "https://news.un.org/example", + }, + screenshot: { + dataUrl: "data:image/png;base64,aGVsbG8=", + fileName: "capture.png", + images: [ + { + filename: "capture.png", + url: "C:\\Users\\Rocky\\AppData\\Local\\Temp\\capture.png", + size: 0, + }, + ], + }, + }, + }); + + assert.equal(result.submitText, [ + "提取并翻译", + "\n提取图片中的文字并翻译成简体中文\n", + "\n\nUN News\n\n\n\nhttps://news.un.org/example\n\n\n\nBanning children from social media is not the answer.\n\n", + ].join("\n\n")); + assert.deepEqual(result.iframeContext, { brainy_request_id: "iframe-123" }); + assert.equal(result.files.length, 1); + assert.equal(result.files[0]?.type, "file"); + assert.equal(result.files[0]?.filename, "capture.png"); + assert.equal(result.files[0]?.mediaType, "image/png"); + assert.ok(result.files[0]?.file instanceof File); + assert.equal(result.files[0]?.file?.name, "capture.png"); + assert.equal(result.files[0]?.file?.type, "image/png"); + assert.equal(result.files[0]?.url, "data:image/png;base64,aGVsbG8="); +}); + +test("keeps screenshot-only iframe submissions in the conversation flow", () => { + const result = parseBrainyChatSubmitMessage({ + source: "brainy-extension", + type: "brainy:chat-submit", + payload: { + screenshot: { + dataUrl: "data:image/jpeg;base64,aGVsbG8=", + images: [ + { + filename: "shot.jpg", + url: "C:\\Users\\Rocky\\AppData\\Local\\Temp\\shot.jpg", + size: 0, + }, + ], + }, + }, + }); + + assert.equal(result.submitText, ""); + assert.equal(result.files.length, 1); + assert.equal(result.files[0]?.filename, "shot.jpg"); + assert.equal(result.files[0]?.mediaType, "image/jpeg"); +}); diff --git a/frontend-web/src/pages/iframe-chat-submit.ts b/frontend-web/src/pages/iframe-chat-submit.ts new file mode 100644 index 00000000..c7583900 --- /dev/null +++ b/frontend-web/src/pages/iframe-chat-submit.ts @@ -0,0 +1,247 @@ +type BrainyPageContext = { + title?: string; + content?: string; + url?: string; +}; + +type BrainyScreenshotPath = { + image_url?: string; +}; + +type BrainyScreenshotImage = { + filename?: string; + url?: string; + size?: number; +}; + +type BrainyScreenshot = { + dataUrl?: string; + fileName?: string; + filePath?: BrainyScreenshotPath[]; + images?: BrainyScreenshotImage[]; +}; + +export type BrainyChatSubmitMessage = { + source?: string; + type?: string; + payload?: { + text?: string; + prompt?: string; + pageContext?: BrainyPageContext; + screenshot?: BrainyScreenshot; + }; + requestId?: string; +}; + +export type IframePromptInputFilePart = { + type: "file"; + url: string; + mediaType: string; + filename: string; + file?: File; +}; + +export type ParsedBrainyChatSubmitMessage = { + submitText: string; + files: IframePromptInputFilePart[]; + iframeContext?: Record; +}; + +function trimToUndefined(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function fileNameFromPath(path: string | undefined): string | undefined { + if (!path) return undefined; + const parts = path.split(/[\\/]/).filter(Boolean); + return parts.at(-1); +} + +function isLikelyLocalPath(path: string): boolean { + return /^[a-zA-Z]:[\\/]/.test(path) || path.startsWith("\\\\"); +} + +function mimeTypeFromFilename(filename: string | undefined): string | undefined { + const lower = filename?.toLowerCase() ?? ""; + if (lower.endsWith(".png")) return "image/png"; + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; + if (lower.endsWith(".webp")) return "image/webp"; + if (lower.endsWith(".gif")) return "image/gif"; + return undefined; +} + +function parseDataUrlMeta(dataUrl: string): { mimeType: string; payload: string } | null { + const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/i.exec(dataUrl); + if (!match) return null; + return { + mimeType: match[1] || "application/octet-stream", + payload: match[2] || "", + }; +} + +function decodeBase64(base64: string): Uint8Array { + if (typeof Buffer !== "undefined") { + return Uint8Array.from(Buffer.from(base64, "base64")); + } + + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i += 1) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +function fileFromDataUrl( + dataUrl: string, + filename: string, + mediaType: string, +): File | undefined { + const meta = parseDataUrlMeta(dataUrl); + if (!meta) return undefined; + + const bytes = decodeBase64(meta.payload); + const arrayBuffer = bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ); + + return new File([arrayBuffer], filename, { + type: mediaType || meta.mimeType, + }); +} + +function normalizePageContext( + pageContext: BrainyPageContext | undefined, +): BrainyPageContext | undefined { + if (!pageContext) return undefined; + + const normalized = { + title: trimToUndefined(pageContext.title), + content: trimToUndefined(pageContext.content), + url: trimToUndefined(pageContext.url), + }; + + return normalized.title || normalized.content || normalized.url + ? normalized + : undefined; +} + +export function buildIframeMessageText({ + text, + prompt, + pageContext, +}: { + text?: string; + prompt?: string; + pageContext?: BrainyPageContext; +}) { + const primaryText = trimToUndefined(text) || trimToUndefined(prompt) || ""; + const sections: string[] = []; + + if (primaryText) { + sections.push(primaryText); + } + + const normalizedPrompt = trimToUndefined(prompt); + if (normalizedPrompt && normalizedPrompt !== primaryText) { + sections.push( + `\n${normalizedPrompt}\n`, + ); + } + + const normalizedPageContext = normalizePageContext(pageContext); + if (normalizedPageContext) { + const pageContextParts = [ + normalizedPageContext.title + ? `\n${normalizedPageContext.title}\n` + : null, + normalizedPageContext.url + ? `\n${normalizedPageContext.url}\n` + : null, + normalizedPageContext.content + ? `\n${normalizedPageContext.content}\n` + : null, + ].filter(Boolean); + + if (pageContextParts.length > 0) { + sections.push( + `\n${pageContextParts.join("\n\n")}\n`, + ); + } + } + + return sections.join("\n\n"); +} + +function buildScreenshotFileParts( + screenshot: BrainyScreenshot | undefined, +): IframePromptInputFilePart[] { + if (!screenshot) return []; + + const fallbackPath = + screenshot.filePath?.map((item) => item.image_url).find(Boolean) ?? + screenshot.images?.map((item) => item.url).find(Boolean); + const fallbackFilename = + trimToUndefined(screenshot.fileName) ?? + screenshot.images?.map((item) => trimToUndefined(item.filename)).find(Boolean) ?? + fileNameFromPath(fallbackPath) ?? + "screenshot.png"; + + const dataUrl = trimToUndefined(screenshot.dataUrl); + if (dataUrl) { + const parsed = parseDataUrlMeta(dataUrl); + const mediaType = + parsed?.mimeType || mimeTypeFromFilename(fallbackFilename) || "image/png"; + const file = fileFromDataUrl(dataUrl, fallbackFilename, mediaType); + + return [ + { + type: "file", + url: dataUrl, + mediaType, + filename: fallbackFilename, + ...(file ? { file } : {}), + }, + ]; + } + + const remoteImage = screenshot.images?.find((item) => { + const url = trimToUndefined(item.url); + return !!url && !isLikelyLocalPath(url); + }); + + if (remoteImage?.url && remoteImage.filename) { + return [ + { + type: "file", + url: remoteImage.url, + mediaType: + mimeTypeFromFilename(remoteImage.filename) || "application/octet-stream", + filename: remoteImage.filename, + }, + ]; + } + + return []; +} + +export function parseBrainyChatSubmitMessage( + data: BrainyChatSubmitMessage, +): ParsedBrainyChatSubmitMessage { + const text = trimToUndefined(data.payload?.text); + const prompt = trimToUndefined(data.payload?.prompt); + const pageContext = normalizePageContext(data.payload?.pageContext); + const submitText = buildIframeMessageText({ text, prompt, pageContext }); + const files = buildScreenshotFileParts(data.payload?.screenshot); + const requestId = trimToUndefined(data.requestId); + + return { + submitText, + files, + ...(requestId + ? { iframeContext: { brainy_request_id: requestId } } + : {}), + }; +}