deerflow-code/frontend-web/src/pages/iframe-chat-submit.ts

type BrainyPageContext = {
  title?: string;
  content?: string;
  url?: string;
};

type BrainyScreenshotPath = {
  image_url?: string;
};

type BrainyScreenshotImage = {
  filename?: string;
  url?: string;
  size?: number;
};

type BrainyScreenshot = {
  dataUrl?: string;
  fileName?: string;
  filePath?: BrainyScreenshotPath[];
  images?: BrainyScreenshotImage[];
};

type BrainyModelSelection = {
  label?: string;
  value?: string;
};

type BrainyUploadedFile = {
  filename?: string;
  size?: string | number;
  path?: string;
  status?: string;
  artifact_url?: string;
};

export type BrainyChatSubmitMessage = {
  source?: string;
  type?: string;
  payload?: {
    text?: string;
    prompt?: string;
    pageContext?: BrainyPageContext;
    screenshot?: BrainyScreenshot;
    model?: BrainyModelSelection;
    uploadedFiles?: BrainyUploadedFile[];
  };
  requestId?: string;
};

export type IframePromptInputFilePart = {
  type: "file";
  url: string;
  mediaType: string;
  filename: string;
  file?: File;
};

export type ParsedBrainyChatSubmitMessage = {
  submitText: string;
  files: IframePromptInputFilePart[];
  iframeContext?: Record<string, unknown>;
  additionalKwargs?: Record<string, unknown>;
};

const IMAGE_PRIORITY_INSTRUCTION = [
  "<attached_image_priority>",
  "A screenshot image is attached for this message.",
  "Use the attached image as the primary source of truth.",
  "Use page context and text prompt only as supporting context.",
  "If the image conflicts with page context or text, trust the image.",
  "</attached_image_priority>",
].join("\n");

function trimToUndefined(value: string | undefined): string | undefined {
  const trimmed = value?.trim();
  return trimmed ? trimmed : undefined;
}

function fileNameFromPath(path: string | undefined): string | undefined {
  if (!path) return undefined;
  const parts = path.split(/[\\/]/).filter(Boolean);
  return parts.at(-1);
}

function isLikelyLocalPath(path: string): boolean {
  return /^[a-zA-Z]:[\\/]/.test(path) || path.startsWith("\\\\");
}

function mimeTypeFromFilename(filename: string | undefined): string | undefined {
  const lower = filename?.toLowerCase() ?? "";
  if (lower.endsWith(".png")) return "image/png";
  if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
  if (lower.endsWith(".webp")) return "image/webp";
  if (lower.endsWith(".gif")) return "image/gif";
  return undefined;
}

function parseDataUrlMeta(dataUrl: string): { mimeType: string; payload: string } | null {
  const match = /^data:([^;,]+)?(?:;base64)?,(.*)$/i.exec(dataUrl);
  if (!match) return null;
  return {
    mimeType: match[1] || "application/octet-stream",
    payload: match[2] || "",
  };
}

function decodeBase64(base64: string): Uint8Array {
  if (typeof Buffer !== "undefined") {
    return Uint8Array.from(Buffer.from(base64, "base64"));
  }

  const binary = atob(base64);
  const bytes = new Uint8Array(binary.length);
  for (let i = 0; i < binary.length; i += 1) {
    bytes[i] = binary.charCodeAt(i);
  }
  return bytes;
}

function fileFromDataUrl(
  dataUrl: string,
  filename: string,
  mediaType: string,
): File | undefined {
  const meta = parseDataUrlMeta(dataUrl);
  if (!meta) return undefined;

  const bytes = decodeBase64(meta.payload);
  const arrayBuffer = bytes.buffer.slice(
    bytes.byteOffset,
    bytes.byteOffset + bytes.byteLength,
  );

  return new File([arrayBuffer], filename, {
    type: mediaType || meta.mimeType,
  });
}

function normalizePageContext(
  pageContext: BrainyPageContext | undefined,
): BrainyPageContext | undefined {
  if (!pageContext) return undefined;

  const normalized = {
    title: trimToUndefined(pageContext.title),
    content: trimToUndefined(pageContext.content),
    url: trimToUndefined(pageContext.url),
  };

  return normalized.title || normalized.content || normalized.url
    ? normalized
    : undefined;
}

function buildIframePromptPrefix({
  prompt,
  pageContext,
  hasAttachedImage = false,
}: {
  prompt?: string;
  pageContext?: BrainyPageContext;
  hasAttachedImage?: boolean;
}) {
  const sections: string[] = [];

  if (hasAttachedImage) {
    sections.push(IMAGE_PRIORITY_INSTRUCTION);
  }

  const normalizedPrompt = trimToUndefined(prompt);
  if (normalizedPrompt) {
    sections.push(
      `<extension_prompt>\n${normalizedPrompt}\n</extension_prompt>`,
    );
  }

  const normalizedPageContext = normalizePageContext(pageContext);
  if (normalizedPageContext) {
    const pageContextParts = [
      normalizedPageContext.title
        ? `<title>\n${normalizedPageContext.title}\n</title>`
        : null,
      normalizedPageContext.url
        ? `<url>\n${normalizedPageContext.url}\n</url>`
        : null,
      normalizedPageContext.content
        ? `<content>\n${normalizedPageContext.content}\n</content>`
        : null,
    ].filter(Boolean);

    if (pageContextParts.length > 0) {
      sections.push(
        `<page_context>\n${pageContextParts.join("\n\n")}\n</page_context>`,
      );
    }
  }

  return sections.join("\n\n");
}

export function buildIframeMessageText({
  text,
  prompt,
}: {
  text?: string;
  prompt?: string;
}) {
  const primaryText = trimToUndefined(text) || trimToUndefined(prompt) || "";
  return primaryText;
}

function buildScreenshotFileParts(
  screenshot: BrainyScreenshot | undefined,
): IframePromptInputFilePart[] {
  if (!screenshot) return [];

  const fallbackPath =
    screenshot.filePath?.map((item) => item.image_url).find(Boolean) ??
    screenshot.images?.map((item) => item.url).find(Boolean);
  const fallbackFilename =
    trimToUndefined(screenshot.fileName) ??
    screenshot.images?.map((item) => trimToUndefined(item.filename)).find(Boolean) ??
    fileNameFromPath(fallbackPath) ??
    "screenshot.png";

  const dataUrl = trimToUndefined(screenshot.dataUrl);
  if (dataUrl) {
    const parsed = parseDataUrlMeta(dataUrl);
    const mediaType =
      parsed?.mimeType || mimeTypeFromFilename(fallbackFilename) || "image/png";
    const file = fileFromDataUrl(dataUrl, fallbackFilename, mediaType);

    return [
      {
        type: "file",
        url: dataUrl,
        mediaType,
        filename: fallbackFilename,
        ...(file ? { file } : {}),
      },
    ];
  }

  const remoteImage = screenshot.images?.find((item) => {
    const url = trimToUndefined(item.url);
    return !!url && !isLikelyLocalPath(url);
  });

  if (remoteImage?.url && remoteImage.filename) {
    return [
      {
        type: "file",
        url: remoteImage.url,
        mediaType:
          mimeTypeFromFilename(remoteImage.filename) || "application/octet-stream",
        filename: remoteImage.filename,
      },
    ];
  }

  return [];
}

function normalizeUploadedFiles(
  uploadedFiles: BrainyUploadedFile[] | undefined,
): Record<string, unknown>[] {
  if (!Array.isArray(uploadedFiles)) {
    return [];
  }

  return uploadedFiles
    .map((file) => {
      const filename = trimToUndefined(file.filename);
      if (!filename) {
        return null;
      }

      const size = String(file.size ?? "0");
      const path = trimToUndefined(file.path);
      const status = trimToUndefined(file.status);
      const artifactUrl = trimToUndefined(file.artifact_url);

      return {
        filename,
        size,
        ...(path ? { path } : {}),
        ...(status ? { status } : {}),
        ...(artifactUrl ? { artifact_url: artifactUrl } : {}),
      };
    })
    .filter((file): file is Record<string, unknown> => file !== null);
}

export function parseBrainyChatSubmitMessage(
  data: BrainyChatSubmitMessage,
): ParsedBrainyChatSubmitMessage {
  const text = trimToUndefined(data.payload?.text);
  const prompt = trimToUndefined(data.payload?.prompt);
  const pageContext = normalizePageContext(data.payload?.pageContext);
  const files = buildScreenshotFileParts(data.payload?.screenshot);
  const submitText = buildIframeMessageText({ text, prompt });
  const promptPrefix = buildIframePromptPrefix({
    prompt,
    pageContext,
    hasAttachedImage: files.length > 0,
  });
  const requestId = trimToUndefined(data.requestId);
  const modelName = trimToUndefined(data.payload?.model?.value);
  const uploadedFiles = normalizeUploadedFiles(data.payload?.uploadedFiles);

  const iframeContext: Record<string, unknown> = {};
  if (requestId) {
    iframeContext.brainy_request_id = requestId;
  }
  if (modelName) {
    iframeContext.model_name = modelName;
  }

  return {
    submitText,
    files,
    ...(Object.keys(iframeContext).length > 0 ? { iframeContext } : {}),
    ...(promptPrefix || uploadedFiles.length > 0
      ? {
          additionalKwargs: {
            ...(promptPrefix ? { prompt_prefix: promptPrefix } : {}),
            ...(uploadedFiles.length > 0 ? { files: uploadedFiles } : {}),
          },
        }
      : {}),
  };
}