Merge branch 'gzy_iframedf' into 'master'

fix: 传图修复

See merge request knowledge-base-bj/deerflow-server!80

frontend-web/src/pages/iframe-chat-submit.test.ts

@@ -32,6 +32,7 @@ test("parses iframe text, page context, and screenshot into a prompt message", (
 
   assert.equal(result.submitText, [
     "提取并翻译",
+    "<attached_image_priority>\nA screenshot image is attached for this message.\nUse the attached image as the primary source of truth.\nUse page context and text prompt only as supporting context.\nIf the image conflicts with page context or text, trust the image.\n</attached_image_priority>",
     "<extension_prompt>\n提取图片中的文字并翻译成简体中文\n</extension_prompt>",
     "<page_context>\n<title>\nUN News\n</title>\n\n<url>\nhttps://news.un.org/example\n</url>\n\n<content>\nBanning children from social media is not the answer.\n</content>\n</page_context>",
   ].join("\n\n"));
@@ -64,7 +65,10 @@ test("keeps screenshot-only iframe submissions in the conversation flow", () =>
     },
   });
 
-  assert.equal(result.submitText, "");
+  assert.equal(
+    result.submitText,
+    "<attached_image_priority>\nA screenshot image is attached for this message.\nUse the attached image as the primary source of truth.\nUse page context and text prompt only as supporting context.\nIf the image conflicts with page context or text, trust the image.\n</attached_image_priority>",
+  );
   assert.equal(result.files.length, 1);
   assert.equal(result.files[0]?.filename, "shot.jpg");
   assert.equal(result.files[0]?.mediaType, "image/jpeg");

frontend-web/src/pages/iframe-chat-submit.ts

@@ -47,6 +47,15 @@ export type ParsedBrainyChatSubmitMessage = {
   iframeContext?: Record<string, unknown>;
 };
 
+const IMAGE_PRIORITY_INSTRUCTION = [
+  "<attached_image_priority>",
+  "A screenshot image is attached for this message.",
+  "Use the attached image as the primary source of truth.",
+  "Use page context and text prompt only as supporting context.",
+  "If the image conflicts with page context or text, trust the image.",
+  "</attached_image_priority>",
+].join("\n");
+
 function trimToUndefined(value: string | undefined): string | undefined {
   const trimmed = value?.trim();
   return trimmed ? trimmed : undefined;
@@ -132,10 +141,12 @@ export function buildIframeMessageText({
   text,
   prompt,
   pageContext,
+  hasAttachedImage = false,
 }: {
   text?: string;
   prompt?: string;
   pageContext?: BrainyPageContext;
+  hasAttachedImage?: boolean;
 }) {
   const primaryText = trimToUndefined(text) || trimToUndefined(prompt) || "";
   const sections: string[] = [];
@@ -144,6 +155,10 @@ export function buildIframeMessageText({
     sections.push(primaryText);
   }
 
+  if (hasAttachedImage) {
+    sections.push(IMAGE_PRIORITY_INSTRUCTION);
+  }
+
   const normalizedPrompt = trimToUndefined(prompt);
   if (normalizedPrompt && normalizedPrompt !== primaryText) {
     sections.push(
@@ -233,8 +248,13 @@ export function parseBrainyChatSubmitMessage(
   const text = trimToUndefined(data.payload?.text);
   const prompt = trimToUndefined(data.payload?.prompt);
   const pageContext = normalizePageContext(data.payload?.pageContext);
-  const submitText = buildIframeMessageText({ text, prompt, pageContext });
   const files = buildScreenshotFileParts(data.payload?.screenshot);
+  const submitText = buildIframeMessageText({
+    text,
+    prompt,
+    pageContext,
+    hasAttachedImage: files.length > 0,
+  });
   const requestId = trimToUndefined(data.requestId);
 
   return {