From 32b708592de06ed6384855d57f4f4bc2c15d8a9a Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Mon, 27 Oct 2025 13:42:41 -0500 Subject: [PATCH] optimize: implement efficient approach for PR #8225 - store base64 directly in backend messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unnecessary memory caching logic - Store base64 data URLs directly in ClineMessage.images[] and ApiMessage content when first received - Eliminate conversion overhead at API call time (base64 already available) - Keep frontend memory efficient with webview URIs for display - Much simpler and more efficient than caching approach - One-time conversion: webview URI → base64 when storing in backend - API calls use pre-stored base64 directly (no file I/O or conversion needed) This achieves PR goals with optimal performance: frontend memory efficiency + instant API calls --- src/core/prompts/responses.ts | 18 ------------------ src/core/task/Task.ts | 23 +++++++++++++++-------- src/integrations/misc/imageDataUrl.ts | 1 + 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/core/prompts/responses.ts b/src/core/prompts/responses.ts index 980c70c4dd..fd51b18fed 100644 --- a/src/core/prompts/responses.ts +++ b/src/core/prompts/responses.ts @@ -3,7 +3,6 @@ import * as path from "path" import * as diff from "diff" import { RooIgnoreController, LOCK_TEXT_SYMBOL } from "../ignore/RooIgnoreController" import { RooProtectedController } from "../protect/RooProtectedController" -import { normalizeImageRefsToDataUrls } from "../../integrations/misc/imageDataUrl" export const formatResponse = { toolDenied: () => `The user denied this operation.`, @@ -201,23 +200,6 @@ const formatImagesIntoBlocks = (images?: string[]): Anthropic.ImageBlockParam[] : [] } -/** - * Async version that converts webview URIs to base64 data URLs before creating image blocks - * This is the missing piece from PR #8225 - allows frontend to use webview URIs while - * backend stores base64 for API calls. - */ -export const formatImagesIntoBlocksAsync = async (images?: string[]): Promise => { - if (!images || images.length === 0) { - return [] - } - - // Convert any webview URIs to base64 data URLs - const dataUrls = await normalizeImageRefsToDataUrls(images) - - // Now use the regular function to create image blocks - return formatImagesIntoBlocks(dataUrls) -} - const toolUseInstructionsReminder = `# Reminder: Instructions for Tool Use Tool uses are formatted using XML-style tags. The tool name itself becomes the XML tag name. Each parameter is enclosed within its own set of tags. Here's the structure: diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 2ae6f694ec..3a83edc7e6 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1212,15 +1212,18 @@ export class Task extends EventEmitter implements TaskLike { await this.providerRef.deref()?.postStateToWebview() - await this.say("text", task, images) + // Convert webview URIs to base64 data URLs for backend storage (one-time conversion) + const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl") + const base64Images = images ? await normalizeImageRefsToDataUrls(images) : undefined + + await this.say("text", task, base64Images) // Store base64 in backend messages this.isInitialized = true - // Convert webview URIs to base64 for backend storage - const { formatImagesIntoBlocksAsync } = await import("../prompts/responses") - let imageBlocks: Anthropic.ImageBlockParam[] = await formatImagesIntoBlocksAsync(images) + // Convert base64 to image blocks for API (no conversion needed, already base64) + const { formatResponse } = await import("../prompts/responses") + let imageBlocks: Anthropic.ImageBlockParam[] = formatResponse.imageBlocks(base64Images) // Task starting - await this.initiateTaskLoop([ { type: "text", @@ -1482,9 +1485,13 @@ export class Task extends EventEmitter implements TaskLike { } if (responseImages && responseImages.length > 0) { - // Convert webview URIs to base64 for backend storage - const { formatImagesIntoBlocksAsync } = await import("../prompts/responses") - const responseImageBlocks = await formatImagesIntoBlocksAsync(responseImages) + // Convert webview URIs to base64 data URLs for backend storage (one-time conversion) + const { normalizeImageRefsToDataUrls } = await import("../../integrations/misc/imageDataUrl") + const base64ResponseImages = await normalizeImageRefsToDataUrls(responseImages) + + // Convert base64 to image blocks for API (no conversion needed, already base64) + const { formatResponse } = await import("../prompts/responses") + const responseImageBlocks = formatResponse.imageBlocks(base64ResponseImages) newUserContent.push(...responseImageBlocks) } diff --git a/src/integrations/misc/imageDataUrl.ts b/src/integrations/misc/imageDataUrl.ts index 354385eb9e..17ded42d14 100644 --- a/src/integrations/misc/imageDataUrl.ts +++ b/src/integrations/misc/imageDataUrl.ts @@ -3,6 +3,7 @@ import * as path from "path" /** * Converts webview URIs to base64 data URLs for API calls. + * Simple fallback for cases where base64 isn't already stored in messages. * This is the missing piece from PR #8225 that allows webview URIs * to be used in frontend while converting to base64 for API calls. */