diff --git a/src/core/prompts/responses.ts b/src/core/prompts/responses.ts index fd51b18fed..980c70c4dd 100644 --- a/src/core/prompts/responses.ts +++ b/src/core/prompts/responses.ts @@ -3,6 +3,7 @@ import * as path from "path" import * as diff from "diff" import { RooIgnoreController, LOCK_TEXT_SYMBOL } from "../ignore/RooIgnoreController" import { RooProtectedController } from "../protect/RooProtectedController" +import { normalizeImageRefsToDataUrls } from "../../integrations/misc/imageDataUrl" export const formatResponse = { toolDenied: () => `The user denied this operation.`, @@ -200,6 +201,23 @@ const formatImagesIntoBlocks = (images?: string[]): Anthropic.ImageBlockParam[] : [] } +/** + * Async version that converts webview URIs to base64 data URLs before creating image blocks + * This is the missing piece from PR #8225 - allows frontend to use webview URIs while + * backend stores base64 for API calls. + */ +export const formatImagesIntoBlocksAsync = async (images?: string[]): Promise => { + if (!images || images.length === 0) { + return [] + } + + // Convert any webview URIs to base64 data URLs + const dataUrls = await normalizeImageRefsToDataUrls(images) + + // Now use the regular function to create image blocks + return formatImagesIntoBlocks(dataUrls) +} + const toolUseInstructionsReminder = `# Reminder: Instructions for Tool Use Tool uses are formatted using XML-style tags. The tool name itself becomes the XML tag name. Each parameter is enclosed within its own set of tags. Here's the structure: diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 2dd9e55c0b..2ae6f694ec 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1215,7 +1215,9 @@ export class Task extends EventEmitter implements TaskLike { await this.say("text", task, images) this.isInitialized = true - let imageBlocks: Anthropic.ImageBlockParam[] = formatResponse.imageBlocks(images) + // Convert webview URIs to base64 for backend storage + const { formatImagesIntoBlocksAsync } = await import("../prompts/responses") + let imageBlocks: Anthropic.ImageBlockParam[] = await formatImagesIntoBlocksAsync(images) // Task starting @@ -1480,7 +1482,10 @@ export class Task extends EventEmitter implements TaskLike { } if (responseImages && responseImages.length > 0) { - newUserContent.push(...formatResponse.imageBlocks(responseImages)) + // Convert webview URIs to base64 for backend storage + const { formatImagesIntoBlocksAsync } = await import("../prompts/responses") + const responseImageBlocks = await formatImagesIntoBlocksAsync(responseImages) + newUserContent.push(...responseImageBlocks) } // Ensure we have at least some content to send to the API. diff --git a/src/integrations/misc/__tests__/imageDataUrl.spec.ts b/src/integrations/misc/__tests__/imageDataUrl.spec.ts new file mode 100644 index 0000000000..4993303034 --- /dev/null +++ b/src/integrations/misc/__tests__/imageDataUrl.spec.ts @@ -0,0 +1,63 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { normalizeImageRefsToDataUrls } from "../imageDataUrl" +import * as fs from "fs/promises" + +// Mock fs module +vi.mock("fs/promises") + +describe("normalizeImageRefsToDataUrls", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it("should pass through data URLs unchanged", async () => { + const dataUrl = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" + const result = await normalizeImageRefsToDataUrls([dataUrl]) + + expect(result).toEqual([dataUrl]) + }) + + it("should convert webview URIs to data URLs", async () => { + const webviewUri = "file:///path/to/test.png" + const mockBuffer = Buffer.from("test image data") + + vi.mocked(fs.readFile).mockResolvedValue(mockBuffer) + + const result = await normalizeImageRefsToDataUrls([webviewUri]) + + expect(result).toHaveLength(1) + expect(result[0]).toMatch(/^data:image\/png;base64,/) + expect(fs.readFile).toHaveBeenCalledWith("/path/to/test.png") + }) + + it("should handle mixed arrays of data URLs and webview URIs", async () => { + const dataUrl = "data:image/jpeg;base64,test123" + const webviewUri = "file:///path/to/test.png" + const mockBuffer = Buffer.from("test image data") + + vi.mocked(fs.readFile).mockResolvedValue(mockBuffer) + + const result = await normalizeImageRefsToDataUrls([dataUrl, webviewUri]) + + expect(result).toHaveLength(2) + expect(result[0]).toBe(dataUrl) // Data URL unchanged + expect(result[1]).toMatch(/^data:image\/png;base64,/) // Webview URI converted + }) + + it("should handle errors gracefully by skipping problematic images", async () => { + const validDataUrl = "data:image/png;base64,valid" + const invalidWebviewUri = "file:///nonexistent/test.png" + + vi.mocked(fs.readFile).mockRejectedValue(new Error("File not found")) + + const result = await normalizeImageRefsToDataUrls([validDataUrl, invalidWebviewUri]) + + expect(result).toEqual([validDataUrl]) // Only valid ones returned + }) + + it("should handle empty arrays", async () => { + const result = await normalizeImageRefsToDataUrls([]) + expect(result).toEqual([]) + }) +}) diff --git a/src/integrations/misc/imageDataUrl.ts b/src/integrations/misc/imageDataUrl.ts new file mode 100644 index 0000000000..354385eb9e --- /dev/null +++ b/src/integrations/misc/imageDataUrl.ts @@ -0,0 +1,94 @@ +import * as fs from "fs/promises" +import * as path from "path" + +/** + * Converts webview URIs to base64 data URLs for API calls. + * This is the missing piece from PR #8225 that allows webview URIs + * to be used in frontend while converting to base64 for API calls. + */ +export async function normalizeImageRefsToDataUrls(imageRefs: string[]): Promise { + const results: string[] = [] + + for (const imageRef of imageRefs) { + // If it's already a data URL, keep it as is + if (imageRef.startsWith("data:image/")) { + results.push(imageRef) + continue + } + + // Convert webview URI to file path and then to base64 + try { + const filePath = webviewUriToFilePath(imageRef) + const buffer = await fs.readFile(filePath) + const base64 = buffer.toString("base64") + const mimeType = getMimeTypeFromPath(filePath) + const dataUrl = `data:${mimeType};base64,${base64}` + results.push(dataUrl) + } catch (error) { + console.error("Failed to convert webview URI to base64:", imageRef, error) + // Skip this image + } + } + + return results +} + +/** + * Converts a webview URI to a file system path + */ +function webviewUriToFilePath(webviewUri: string): string { + // Handle vscode-resource URIs like: + // vscode-resource://vscode-webview/path/to/file + if (webviewUri.includes("vscode-resource://")) { + // Extract the path portion after vscode-resource://vscode-webview/ + const match = webviewUri.match(/vscode-resource:\/\/[^\/]+(.+)/) + if (match) { + return decodeURIComponent(match[1]) + } + } + + // Handle file:// URIs + if (webviewUri.startsWith("file://")) { + return decodeURIComponent(webviewUri.substring(7)) + } + + // Handle VS Code webview URIs that contain encoded paths + if (webviewUri.includes("vscode-userdata") || webviewUri.includes("vscode-cdn.net")) { + // Try to decode the URI and extract the file path + const decoded = decodeURIComponent(webviewUri) + // Look for a file path pattern in the decoded URI + const pathMatch = decoded.match(/(?:Users|C:)([^?#]+\.(?:png|jpg|jpeg|gif|webp))/i) + if (pathMatch) { + const extractedPath = pathMatch[0] + return extractedPath + } + } + + // As a last resort, try treating it as a file path + return webviewUri +} + +/** + * Gets the MIME type from a file path + */ +function getMimeTypeFromPath(filePath: string): string { + const ext = path.extname(filePath).toLowerCase() + + switch (ext) { + case ".png": + return "image/png" + case ".jpg": + case ".jpeg": + return "image/jpeg" + case ".gif": + return "image/gif" + case ".webp": + return "image/webp" + case ".svg": + return "image/svg+xml" + case ".bmp": + return "image/bmp" + default: + return "image/png" + } +}