From 8ba3a674012e381d4ad897550bddc49842066b7d Mon Sep 17 00:00:00 2001 From: ZyGuo Date: Wed, 3 Jun 2026 11:12:10 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BC=A0=E5=9B=BE=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pages/iframe-chat-submit.test.ts | 6 ++++- frontend-web/src/pages/iframe-chat-submit.ts | 22 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/frontend-web/src/pages/iframe-chat-submit.test.ts b/frontend-web/src/pages/iframe-chat-submit.test.ts index 690570fd..1b49c128 100644 --- a/frontend-web/src/pages/iframe-chat-submit.test.ts +++ b/frontend-web/src/pages/iframe-chat-submit.test.ts @@ -32,6 +32,7 @@ test("parses iframe text, page context, and screenshot into a prompt message", ( assert.equal(result.submitText, [ "提取并翻译", + "\nA screenshot image is attached for this message.\nUse the attached image as the primary source of truth.\nUse page context and text prompt only as supporting context.\nIf the image conflicts with page context or text, trust the image.\n", "\n提取图片中的文字并翻译成简体中文\n", "\n\nUN News\n\n\n\nhttps://news.un.org/example\n\n\n\nBanning children from social media is not the answer.\n\n", ].join("\n\n")); @@ -64,7 +65,10 @@ test("keeps screenshot-only iframe submissions in the conversation flow", () => }, }); - assert.equal(result.submitText, ""); + assert.equal( + result.submitText, + "\nA screenshot image is attached for this message.\nUse the attached image as the primary source of truth.\nUse page context and text prompt only as supporting context.\nIf the image conflicts with page context or text, trust the image.\n", + ); assert.equal(result.files.length, 1); assert.equal(result.files[0]?.filename, "shot.jpg"); assert.equal(result.files[0]?.mediaType, "image/jpeg"); diff --git a/frontend-web/src/pages/iframe-chat-submit.ts b/frontend-web/src/pages/iframe-chat-submit.ts index c7583900..981f0634 100644 --- a/frontend-web/src/pages/iframe-chat-submit.ts +++ b/frontend-web/src/pages/iframe-chat-submit.ts @@ -47,6 +47,15 @@ export type ParsedBrainyChatSubmitMessage = { iframeContext?: Record; }; +const IMAGE_PRIORITY_INSTRUCTION = [ + "", + "A screenshot image is attached for this message.", + "Use the attached image as the primary source of truth.", + "Use page context and text prompt only as supporting context.", + "If the image conflicts with page context or text, trust the image.", + "", +].join("\n"); + function trimToUndefined(value: string | undefined): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; @@ -132,10 +141,12 @@ export function buildIframeMessageText({ text, prompt, pageContext, + hasAttachedImage = false, }: { text?: string; prompt?: string; pageContext?: BrainyPageContext; + hasAttachedImage?: boolean; }) { const primaryText = trimToUndefined(text) || trimToUndefined(prompt) || ""; const sections: string[] = []; @@ -144,6 +155,10 @@ export function buildIframeMessageText({ sections.push(primaryText); } + if (hasAttachedImage) { + sections.push(IMAGE_PRIORITY_INSTRUCTION); + } + const normalizedPrompt = trimToUndefined(prompt); if (normalizedPrompt && normalizedPrompt !== primaryText) { sections.push( @@ -233,8 +248,13 @@ export function parseBrainyChatSubmitMessage( const text = trimToUndefined(data.payload?.text); const prompt = trimToUndefined(data.payload?.prompt); const pageContext = normalizePageContext(data.payload?.pageContext); - const submitText = buildIframeMessageText({ text, prompt, pageContext }); const files = buildScreenshotFileParts(data.payload?.screenshot); + const submitText = buildIframeMessageText({ + text, + prompt, + pageContext, + hasAttachedImage: files.length > 0, + }); const requestId = trimToUndefined(data.requestId); return {