From d39a4e1294acdcca25e339e6a5cf5e868770e51a Mon Sep 17 00:00:00 2001 From: Roo Code Date: Thu, 12 Mar 2026 01:15:03 +0000 Subject: [PATCH] feat: add configurable image downscaling before LLM upload Adds a new image downscaling feature that resizes images before sending them to the LLM, reducing token usage for high-resolution screenshots. Changes: - Add sharp dependency for image processing - New settings: maxImageDimension (0=disabled) and imageDownscaleQuality (1-100) - New resize-image.ts utility using sharp for proportional downscaling - Integrate resizing into all image entry points: selectImages, readImageAsDataUrlWithBuffer, processImageFile, resolveImageMentions - Add settings UI controls in Context Management settings tab - Add English translation strings for the new settings - Downscaling disabled by default (opt-in) Closes #11912 --- packages/types/src/global-settings.ts | 14 ++ packages/types/src/vscode-extension-host.ts | 2 + pnpm-lock.yaml | 53 +++--- .../__tests__/resolveImageMentions.spec.ts | 5 +- src/core/mentions/resolveImageMentions.ts | 11 +- src/core/tools/ReadFileTool.ts | 18 +- src/core/tools/helpers/imageHelpers.ts | 44 +++-- src/core/webview/ClineProvider.ts | 6 + src/core/webview/webviewMessageHandler.ts | 11 +- .../misc/__tests__/resize-image.spec.ts | 162 ++++++++++++++++++ src/integrations/misc/process-images.ts | 30 +++- src/integrations/misc/resize-image.ts | 99 +++++++++++ src/package.json | 2 + .../settings/ContextManagementSettings.tsx | 66 +++++++ .../src/components/settings/SettingsView.tsx | 6 + webview-ui/src/i18n/locales/en/settings.json | 10 ++ 16 files changed, 491 insertions(+), 48 deletions(-) create mode 100644 src/integrations/misc/__tests__/resize-image.spec.ts create mode 100644 src/integrations/misc/resize-image.ts diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index 288f6c2118..961667c7e0 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -167,6 +167,20 @@ export const globalSettingsSchema = z.object({ maxImageFileSize: z.number().optional(), maxTotalImageSize: z.number().optional(), + /** + * Maximum dimension (width or height) in pixels for image downscaling before LLM upload. + * Images exceeding this dimension are proportionally resized, preserving aspect ratio. + * Set to 0 (default) to disable downscaling. + */ + maxImageDimension: z.number().min(0).optional(), + + /** + * JPEG/WebP quality (1-100) used when re-encoding resized images. + * Only applies when maxImageDimension is set and the image is downscaled. + * @default 85 + */ + imageDownscaleQuality: z.number().min(1).max(100).optional(), + terminalOutputPreviewSize: z.enum(["small", "medium", "large"]).optional(), terminalShellIntegrationTimeout: z.number().optional(), terminalShellIntegrationDisabled: z.boolean().optional(), diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index b20539afe4..963c2c14e7 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -330,6 +330,8 @@ export type ExtensionState = Pick< maxReadFileLine?: number // Maximum line limit for read_file tool (-1 for default) maxImageFileSize: number // Maximum size of image files to process in MB maxTotalImageSize: number // Maximum total size for all images in a single read operation in MB + maxImageDimension?: number // Maximum dimension (width or height) in pixels for image downscaling (0 = disabled) + imageDownscaleQuality?: number // JPEG/WebP quality (1-100) for re-encoding resized images (default 85) experiments: Experiments // Map of experiment IDs to their enabled state diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d95c2f0234..7fa78f7545 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -962,6 +962,9 @@ importers: serialize-error: specifier: ^12.0.0 version: 12.0.0 + sharp: + specifier: ^0.34.5 + version: 0.34.5 shell-quote: specifier: ^1.8.2 version: 1.8.3 @@ -1074,6 +1077,9 @@ importers: '@types/semver-compare': specifier: ^1.0.3 version: 1.0.3 + '@types/sharp': + specifier: ^0.32.0 + version: 0.32.0 '@types/shell-quote': specifier: ^1.7.5 version: 1.7.5 @@ -2027,9 +2033,6 @@ packages: '@emnapi/core@1.4.3': resolution: {integrity: sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g==} - '@emnapi/runtime@1.4.3': - resolution: {integrity: sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ==} - '@emnapi/runtime@1.8.1': resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==} @@ -4585,6 +4588,10 @@ packages: '@types/semver-compare@1.0.3': resolution: {integrity: sha512-mVZkB2QjXmZhh+MrtwMlJ8BqUnmbiSkpd88uOWskfwB8yitBT0tBRAKt+41VRgZD9zr9Sc+Xs02qGgvzd1Rq/Q==} + '@types/sharp@0.32.0': + resolution: {integrity: sha512-OOi3kL+FZDnPhVzsfD37J88FNeZh6gQsGcLc95NbeURRGvmSjeXiDcyWzF2o3yh/gQAUn2uhh/e+CPCa5nwAxw==} + deprecated: This is a stub types definition. sharp provides its own type definitions, so you do not need this installed. + '@types/shell-quote@1.7.5': resolution: {integrity: sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==} @@ -5130,6 +5137,7 @@ packages: basic-ftp@5.0.5: resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==} engines: {node: '>=10.0.0'} + deprecated: Security vulnerability fixed in 5.2.0, please upgrade better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} @@ -5957,10 +5965,6 @@ packages: resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} engines: {node: '>=8'} - detect-libc@2.0.4: - resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==} - engines: {node: '>=8'} - detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -8976,6 +8980,7 @@ packages: prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true prelude-ls@1.2.1: @@ -12149,11 +12154,6 @@ snapshots: tslib: 2.8.1 optional: true - '@emnapi/runtime@1.4.3': - dependencies: - tslib: 2.8.1 - optional: true - '@emnapi/runtime@1.8.1': dependencies: tslib: 2.8.1 @@ -12391,8 +12391,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@img/colour@1.0.0': - optional: true + '@img/colour@1.0.0': {} '@img/sharp-darwin-arm64@0.34.5': optionalDependencies: @@ -12733,14 +12732,14 @@ snapshots: '@napi-rs/wasm-runtime@0.2.10': dependencies: '@emnapi/core': 1.4.3 - '@emnapi/runtime': 1.4.3 + '@emnapi/runtime': 1.8.1 '@tybys/wasm-util': 0.9.0 optional: true '@napi-rs/wasm-runtime@0.2.11': dependencies: '@emnapi/core': 1.4.3 - '@emnapi/runtime': 1.4.3 + '@emnapi/runtime': 1.8.1 '@tybys/wasm-util': 0.9.0 optional: true @@ -14319,7 +14318,7 @@ snapshots: '@tailwindcss/oxide@4.1.6': dependencies: - detect-libc: 2.0.4 + detect-libc: 2.1.2 tar: 7.4.3 optionalDependencies: '@tailwindcss/oxide-android-arm64': 4.1.6 @@ -14337,7 +14336,7 @@ snapshots: '@tailwindcss/oxide@4.1.8': dependencies: - detect-libc: 2.0.4 + detect-libc: 2.1.2 tar: 7.4.3 optionalDependencies: '@tailwindcss/oxide-android-arm64': 4.1.8 @@ -14720,6 +14719,10 @@ snapshots: '@types/semver-compare@1.0.3': {} + '@types/sharp@0.32.0': + dependencies: + sharp: 0.34.5 + '@types/shell-quote@1.7.5': {} '@types/stack-utils@2.0.3': {} @@ -14974,7 +14977,7 @@ snapshots: sirv: 3.0.1 tinyglobby: 0.2.14 tinyrainbow: 2.0.0 - vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.2.1)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) + vitest: 3.2.4(@types/debug@4.1.12)(@types/node@20.17.50)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) '@vitest/utils@3.2.4': dependencies: @@ -16253,10 +16256,7 @@ snapshots: detect-libc@2.0.2: optional: true - detect-libc@2.0.4: {} - - detect-libc@2.1.2: - optional: true + detect-libc@2.1.2: {} detect-node-es@1.1.0: {} @@ -18333,7 +18333,7 @@ snapshots: lightningcss@1.29.2: dependencies: - detect-libc: 2.0.4 + detect-libc: 2.1.2 optionalDependencies: lightningcss-darwin-arm64: 1.29.2 lightningcss-darwin-x64: 1.29.2 @@ -18348,7 +18348,7 @@ snapshots: lightningcss@1.30.1: dependencies: - detect-libc: 2.0.4 + detect-libc: 2.1.2 optionalDependencies: lightningcss-darwin-arm64: 1.30.1 lightningcss-darwin-x64: 1.30.1 @@ -19761,7 +19761,7 @@ snapshots: prebuild-install@7.1.3: dependencies: - detect-libc: 2.0.4 + detect-libc: 2.1.2 expand-template: 2.0.3 github-from-package: 0.0.0 minimist: 1.2.8 @@ -20639,7 +20639,6 @@ snapshots: '@img/sharp-win32-arm64': 0.34.5 '@img/sharp-win32-ia32': 0.34.5 '@img/sharp-win32-x64': 0.34.5 - optional: true shebang-command@2.0.0: dependencies: diff --git a/src/core/mentions/__tests__/resolveImageMentions.spec.ts b/src/core/mentions/__tests__/resolveImageMentions.spec.ts index 747c778819..4fe55af881 100644 --- a/src/core/mentions/__tests__/resolveImageMentions.spec.ts +++ b/src/core/mentions/__tests__/resolveImageMentions.spec.ts @@ -41,7 +41,10 @@ describe("resolveImageMentions", () => { }) expect(mockValidateImage).toHaveBeenCalled() - expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png")) + expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"), { + maxDimension: undefined, + quality: undefined, + }) expect(result.text).toBe("Please look at @/assets/cat.png") expect(result.images).toEqual([dataUrl]) }) diff --git a/src/core/mentions/resolveImageMentions.ts b/src/core/mentions/resolveImageMentions.ts index 0a0344348f..4e808b0130 100644 --- a/src/core/mentions/resolveImageMentions.ts +++ b/src/core/mentions/resolveImageMentions.ts @@ -23,6 +23,10 @@ export interface ResolveImageMentionsOptions { maxImageFileSize?: number /** Maximum total size of all images in MB. Defaults to 20MB. */ maxTotalImageSize?: number + /** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */ + maxImageDimension?: number + /** JPEG/WebP quality (1-100) for re-encoding resized images. */ + imageDownscaleQuality?: number } export interface ResolveImageMentionsResult { @@ -65,6 +69,8 @@ export async function resolveImageMentions({ supportsImages = true, maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB, maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, + maxImageDimension, + imageDownscaleQuality, }: ResolveImageMentionsOptions): Promise { const existingImages = Array.isArray(images) ? images : [] if (existingImages.length >= MAX_IMAGES_PER_MESSAGE) { @@ -127,7 +133,10 @@ export async function resolveImageMentions({ continue } - const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath) + const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath, { + maxDimension: maxImageDimension, + quality: imageDownscaleQuality, + }) newImages.push(dataUrl) // Track memory usage diff --git a/src/core/tools/ReadFileTool.ts b/src/core/tools/ReadFileTool.ts index 8ad6a3b33d..d3d1274462 100644 --- a/src/core/tools/ReadFileTool.ts +++ b/src/core/tools/ReadFileTool.ts @@ -171,6 +171,8 @@ export class ReadFileTool extends BaseTool<"read_file"> { const { maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB, maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, + maxImageDimension, + imageDownscaleQuality, } = state ?? {} for (const fileResult of fileResults) { @@ -207,6 +209,8 @@ export class ReadFileTool extends BaseTool<"read_file"> { maxTotalImageSize, imageMemoryTracker, updateFileResult, + maxImageDimension, + imageDownscaleQuality, ) continue } @@ -341,6 +345,8 @@ export class ReadFileTool extends BaseTool<"read_file"> { maxTotalImageSize: number, imageMemoryTracker: ImageMemoryTracker, updateFileResult: (path: string, updates: Partial) => void, + maxImageDimension?: number, + imageDownscaleQuality?: number, ): Promise { const fileExtension = path.extname(relPath).toLowerCase() const supportedBinaryFormats = getSupportedBinaryFormats() @@ -364,7 +370,10 @@ export class ReadFileTool extends BaseTool<"read_file"> { return } - const imageResult = await processImageFile(fullPath) + const imageResult = await processImageFile(fullPath, { + maxDimension: maxImageDimension, + quality: imageDownscaleQuality, + }) imageMemoryTracker.addMemoryUsage(imageResult.sizeInMB) await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource) @@ -744,6 +753,8 @@ export class ReadFileTool extends BaseTool<"read_file"> { const { maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB, maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB, + maxImageDimension: legacyMaxDim, + imageDownscaleQuality: legacyQuality, } = state ?? {} const validation = await validateImageForProcessing( fullPath, @@ -756,7 +767,10 @@ export class ReadFileTool extends BaseTool<"read_file"> { results.push(`File: ${relPath}\nNotice: ${validation.notice ?? "Image validation failed"}`) continue } - const imageResult = await processImageFile(fullPath) + const imageResult = await processImageFile(fullPath, { + maxDimension: legacyMaxDim, + quality: legacyQuality, + }) if (imageResult) { results.push(`File: ${relPath}\n[Image file - content processed for vision model]`) } diff --git a/src/core/tools/helpers/imageHelpers.ts b/src/core/tools/helpers/imageHelpers.ts index a1adb078e6..7c7e1c30d2 100644 --- a/src/core/tools/helpers/imageHelpers.ts +++ b/src/core/tools/helpers/imageHelpers.ts @@ -2,6 +2,7 @@ import path from "path" import * as fs from "fs/promises" import { t } from "../../../i18n" import prettyBytes from "pretty-bytes" +import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "../../../integrations/misc/resize-image" /** * Default maximum allowed image file size in bytes (5MB) @@ -69,15 +70,37 @@ export interface ImageProcessingResult { notice: string } -/** - * Reads an image file and returns both the data URL and buffer - */ -export async function readImageAsDataUrlWithBuffer(filePath: string): Promise<{ dataUrl: string; buffer: Buffer }> { - const fileBuffer = await fs.readFile(filePath) - const base64 = fileBuffer.toString("base64") - const ext = path.extname(filePath).toLowerCase() +export interface ReadImageOptions { + /** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */ + maxDimension?: number + /** JPEG/WebP quality (1-100) for re-encoding resized images. */ + quality?: number +} +/** + * Reads an image file and returns both the data URL and buffer. + * Optionally downscales the image if maxDimension is set. + */ +export async function readImageAsDataUrlWithBuffer( + filePath: string, + options?: ReadImageOptions, +): Promise<{ dataUrl: string; buffer: Buffer }> { + let fileBuffer = await fs.readFile(filePath) + const ext = path.extname(filePath).toLowerCase() const mimeType = IMAGE_MIME_TYPES[ext] || "image/png" + + // Downscale if configured + if (options?.maxDimension && options.maxDimension > 0) { + const resizeResult = await maybeResizeImage({ + buffer: fileBuffer, + mimeType, + maxDimension: options.maxDimension, + quality: options.quality ?? DEFAULT_IMAGE_DOWNSCALE_QUALITY, + }) + fileBuffer = resizeResult.buffer + } + + const base64 = fileBuffer.toString("base64") const dataUrl = `data:${mimeType};base64,${base64}` return { dataUrl, buffer: fileBuffer } @@ -145,11 +168,12 @@ export async function validateImageForProcessing( } /** - * Processes an image file and returns the result + * Processes an image file and returns the result. + * Optionally downscales the image if resize options are provided. */ -export async function processImageFile(fullPath: string): Promise { +export async function processImageFile(fullPath: string, options?: ReadImageOptions): Promise { const imageStats = await fs.stat(fullPath) - const { dataUrl, buffer } = await readImageAsDataUrlWithBuffer(fullPath) + const { dataUrl, buffer } = await readImageAsDataUrlWithBuffer(fullPath, options) const imageSizeInKB = Math.round(imageStats.size / 1024) const imageSizeInMB = imageStats.size / (1024 * 1024) const noticeText = t("tools:readFile.imageWithSize", { size: imageSizeInKB }) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 7bd969e52d..ae488f12be 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -2182,6 +2182,8 @@ export class ClineProvider language, maxImageFileSize, maxTotalImageSize, + maxImageDimension, + imageDownscaleQuality, historyPreviewCollapsed, reasoningBlockCollapsed, enterBehavior, @@ -2307,6 +2309,8 @@ export class ClineProvider renderContext: this.renderContext, maxImageFileSize: maxImageFileSize ?? 5, maxTotalImageSize: maxTotalImageSize ?? 20, + maxImageDimension: maxImageDimension ?? 0, + imageDownscaleQuality: imageDownscaleQuality ?? 85, settingsImportedAt: this.settingsImportedAt, historyPreviewCollapsed: historyPreviewCollapsed ?? false, reasoningBlockCollapsed: reasoningBlockCollapsed ?? true, @@ -2531,6 +2535,8 @@ export class ClineProvider enableSubfolderRules: stateValues.enableSubfolderRules ?? false, maxImageFileSize: stateValues.maxImageFileSize ?? 5, maxTotalImageSize: stateValues.maxTotalImageSize ?? 20, + maxImageDimension: stateValues.maxImageDimension ?? 0, + imageDownscaleQuality: stateValues.imageDownscaleQuality ?? 85, historyPreviewCollapsed: stateValues.historyPreviewCollapsed ?? false, reasoningBlockCollapsed: stateValues.reasoningBlockCollapsed ?? true, enterBehavior: stateValues.enterBehavior ?? "send", diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index d27fd6bec0..b06c209c1a 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -185,6 +185,8 @@ export const webviewMessageHandler = async ( rooIgnoreController: currentTask?.rooIgnoreController, maxImageFileSize: state.maxImageFileSize, maxTotalImageSize: state.maxTotalImageSize, + maxImageDimension: state.maxImageDimension, + imageDownscaleQuality: state.imageDownscaleQuality, }) return resolved } @@ -766,8 +768,12 @@ export const webviewMessageHandler = async ( await updateGlobalState("lastShownAnnouncementId", provider.latestAnnouncementId) await provider.postStateToWebview() break - case "selectImages": - const images = await selectImages() + case "selectImages": { + const selectState = await provider.getState() + const images = await selectImages({ + maxDimension: selectState.maxImageDimension, + quality: selectState.imageDownscaleQuality, + }) await provider.postMessageToWebview({ type: "selectedImages", images, @@ -775,6 +781,7 @@ export const webviewMessageHandler = async ( messageTs: message.messageTs, }) break + } case "exportCurrentTask": const currentTaskId = provider.getCurrentTask()?.taskId if (currentTaskId) { diff --git a/src/integrations/misc/__tests__/resize-image.spec.ts b/src/integrations/misc/__tests__/resize-image.spec.ts new file mode 100644 index 0000000000..8e4a7df8d5 --- /dev/null +++ b/src/integrations/misc/__tests__/resize-image.spec.ts @@ -0,0 +1,162 @@ +import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "../resize-image" +import sharp from "sharp" + +describe("maybeResizeImage", () => { + /** + * Helper to create a test image buffer with specified dimensions. + */ + async function createTestImage( + width: number, + height: number, + format: "png" | "jpeg" | "webp" = "png", + ): Promise { + const channels = 3 + const rawData = Buffer.alloc(width * height * channels, 128) + let pipeline = sharp(rawData, { raw: { width, height, channels } }) + + if (format === "png") { + pipeline = pipeline.png() + } else if (format === "jpeg") { + pipeline = pipeline.jpeg() + } else if (format === "webp") { + pipeline = pipeline.webp() + } + + return pipeline.toBuffer() + } + + it("should return original buffer when maxDimension is 0", async () => { + const buffer = await createTestImage(100, 100) + const result = await maybeResizeImage({ + buffer, + mimeType: "image/png", + maxDimension: 0, + }) + expect(result.wasResized).toBe(false) + expect(result.buffer).toBe(buffer) // same reference + }) + + it("should return original buffer when maxDimension is undefined", async () => { + const buffer = await createTestImage(100, 100) + const result = await maybeResizeImage({ + buffer, + mimeType: "image/png", + }) + expect(result.wasResized).toBe(false) + expect(result.buffer).toBe(buffer) + }) + + it("should not resize when image is smaller than maxDimension", async () => { + const buffer = await createTestImage(200, 100) + const result = await maybeResizeImage({ + buffer, + mimeType: "image/png", + maxDimension: 300, + }) + expect(result.wasResized).toBe(false) + expect(result.buffer).toBe(buffer) + }) + + it("should downscale a wide image exceeding maxDimension", async () => { + const buffer = await createTestImage(2000, 1000, "png") + const result = await maybeResizeImage({ + buffer, + mimeType: "image/png", + maxDimension: 500, + }) + expect(result.wasResized).toBe(true) + + const metadata = await sharp(result.buffer).metadata() + expect(metadata.width).toBeLessThanOrEqual(500) + expect(metadata.height).toBeLessThanOrEqual(500) + // Check aspect ratio is roughly preserved (2:1) + expect(metadata.width! / metadata.height!).toBeCloseTo(2, 0) + }) + + it("should downscale a tall image exceeding maxDimension", async () => { + const buffer = await createTestImage(500, 2000, "jpeg") + const result = await maybeResizeImage({ + buffer, + mimeType: "image/jpeg", + maxDimension: 1000, + }) + expect(result.wasResized).toBe(true) + + const metadata = await sharp(result.buffer).metadata() + expect(metadata.width).toBeLessThanOrEqual(1000) + expect(metadata.height).toBeLessThanOrEqual(1000) + // Check aspect ratio is roughly preserved (1:4) + expect(metadata.height! / metadata.width!).toBeCloseTo(4, 0) + }) + + it("should handle webp format", async () => { + const buffer = await createTestImage(1500, 1500, "webp") + const result = await maybeResizeImage({ + buffer, + mimeType: "image/webp", + maxDimension: 800, + }) + expect(result.wasResized).toBe(true) + + const metadata = await sharp(result.buffer).metadata() + expect(metadata.format).toBe("webp") + expect(metadata.width).toBeLessThanOrEqual(800) + expect(metadata.height).toBeLessThanOrEqual(800) + }) + + it("should return original buffer for unsupported mime types", async () => { + const buffer = await createTestImage(2000, 2000) + const result = await maybeResizeImage({ + buffer, + mimeType: "image/gif", + maxDimension: 500, + }) + expect(result.wasResized).toBe(false) + expect(result.buffer).toBe(buffer) + }) + + it("should use default quality when not specified", async () => { + expect(DEFAULT_IMAGE_DOWNSCALE_QUALITY).toBe(85) + }) + + it("should accept custom quality setting for jpeg without error", async () => { + const buffer = await createTestImage(2000, 2000, "jpeg") + + const resultHighQ = await maybeResizeImage({ + buffer, + mimeType: "image/jpeg", + maxDimension: 500, + quality: 95, + }) + + const resultLowQ = await maybeResizeImage({ + buffer, + mimeType: "image/jpeg", + maxDimension: 500, + quality: 10, + }) + + // Both should be resized successfully + expect(resultHighQ.wasResized).toBe(true) + expect(resultLowQ.wasResized).toBe(true) + + // Both should produce valid image buffers + const metaHigh = await sharp(resultHighQ.buffer).metadata() + const metaLow = await sharp(resultLowQ.buffer).metadata() + expect(metaHigh.format).toBe("jpeg") + expect(metaLow.format).toBe("jpeg") + expect(metaHigh.width).toBeLessThanOrEqual(500) + expect(metaLow.width).toBeLessThanOrEqual(500) + }) + + it("should not upscale small images", async () => { + const buffer = await createTestImage(100, 50, "png") + const result = await maybeResizeImage({ + buffer, + mimeType: "image/png", + maxDimension: 500, + }) + expect(result.wasResized).toBe(false) + expect(result.buffer).toBe(buffer) + }) +}) diff --git a/src/integrations/misc/process-images.ts b/src/integrations/misc/process-images.ts index cf3e201538..d10fa97529 100644 --- a/src/integrations/misc/process-images.ts +++ b/src/integrations/misc/process-images.ts @@ -1,9 +1,17 @@ import * as vscode from "vscode" import fs from "fs/promises" import * as path from "path" +import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "./resize-image" -export async function selectImages(): Promise { - const options: vscode.OpenDialogOptions = { +export interface SelectImagesOptions { + /** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */ + maxDimension?: number + /** JPEG/WebP quality (1-100) for re-encoding resized images. */ + quality?: number +} + +export async function selectImages(options?: SelectImagesOptions): Promise { + const dialogOptions: vscode.OpenDialogOptions = { canSelectMany: true, openLabel: "Select", filters: { @@ -11,7 +19,7 @@ export async function selectImages(): Promise { }, } - const fileUris = await vscode.window.showOpenDialog(options) + const fileUris = await vscode.window.showOpenDialog(dialogOptions) if (!fileUris || fileUris.length === 0) { return [] @@ -20,9 +28,21 @@ export async function selectImages(): Promise { return await Promise.all( fileUris.map(async (uri) => { const imagePath = uri.fsPath - const buffer = await fs.readFile(imagePath) - const base64 = buffer.toString("base64") + let buffer = await fs.readFile(imagePath) const mimeType = getMimeType(imagePath) + + // Downscale if configured + if (options?.maxDimension && options.maxDimension > 0) { + const resizeResult = await maybeResizeImage({ + buffer, + mimeType, + maxDimension: options.maxDimension, + quality: options.quality ?? DEFAULT_IMAGE_DOWNSCALE_QUALITY, + }) + buffer = resizeResult.buffer + } + + const base64 = buffer.toString("base64") const dataUrl = `data:${mimeType};base64,${base64}` return dataUrl }), diff --git a/src/integrations/misc/resize-image.ts b/src/integrations/misc/resize-image.ts new file mode 100644 index 0000000000..ded2e623ea --- /dev/null +++ b/src/integrations/misc/resize-image.ts @@ -0,0 +1,99 @@ +import sharp from "sharp" + +/** + * Default image downscale quality (JPEG/WebP) when re-encoding resized images. + */ +export const DEFAULT_IMAGE_DOWNSCALE_QUALITY = 85 + +/** + * Mime type to sharp output format mapping. + */ +const MIME_TO_FORMAT: Record = { + "image/png": "png", + "image/jpeg": "jpeg", + "image/webp": "webp", +} + +export interface ResizeImageOptions { + /** The image buffer to potentially resize. */ + buffer: Buffer + /** The MIME type of the image (e.g. "image/png"). */ + mimeType: string + /** Maximum dimension (width or height) in pixels. 0 or undefined means no resizing. */ + maxDimension?: number + /** JPEG/WebP quality (1-100) for re-encoding. Defaults to 85. */ + quality?: number +} + +export interface ResizeImageResult { + /** The (possibly resized) image buffer. */ + buffer: Buffer + /** Whether the image was actually resized. */ + wasResized: boolean +} + +/** + * Conditionally downscales an image buffer if either dimension exceeds `maxDimension`. + * Preserves aspect ratio. Returns the original buffer unchanged if no resizing is needed + * or if the format is unsupported for resizing. + */ +export async function maybeResizeImage({ + buffer, + mimeType, + maxDimension, + quality = DEFAULT_IMAGE_DOWNSCALE_QUALITY, +}: ResizeImageOptions): Promise { + // If downscaling is disabled or dimension is 0/undefined, return as-is + if (!maxDimension || maxDimension <= 0) { + return { buffer, wasResized: false } + } + + // Only resize formats we can handle + const format = MIME_TO_FORMAT[mimeType] + if (!format) { + return { buffer, wasResized: false } + } + + const image = sharp(buffer) + const metadata = await image.metadata() + + if (!metadata.width || !metadata.height) { + return { buffer, wasResized: false } + } + + // Only downscale -- never upscale + if (metadata.width <= maxDimension && metadata.height <= maxDimension) { + return { buffer, wasResized: false } + } + + // Calculate new dimensions preserving aspect ratio + const aspectRatio = metadata.width / metadata.height + let newWidth: number + let newHeight: number + + if (metadata.width >= metadata.height) { + newWidth = maxDimension + newHeight = Math.round(maxDimension / aspectRatio) + } else { + newHeight = maxDimension + newWidth = Math.round(maxDimension * aspectRatio) + } + + // Perform the resize + let pipeline = image.resize(newWidth, newHeight, { + fit: "inside", + withoutEnlargement: true, + }) + + // Re-encode in the original format with quality setting where applicable + if (format === "jpeg") { + pipeline = pipeline.jpeg({ quality }) + } else if (format === "webp") { + pipeline = pipeline.webp({ quality }) + } else if (format === "png") { + pipeline = pipeline.png() + } + + const resizedBuffer = await pipeline.toBuffer() + return { buffer: resizedBuffer, wasResized: true } +} diff --git a/src/package.json b/src/package.json index 7c4889abd8..44de7f4e93 100644 --- a/src/package.json +++ b/src/package.json @@ -519,6 +519,7 @@ "say": "^0.16.0", "semver-compare": "^1.0.0", "serialize-error": "^12.0.0", + "sharp": "^0.34.5", "shell-quote": "^1.8.2", "simple-git": "^3.27.0", "sound-play": "^1.1.0", @@ -558,6 +559,7 @@ "@types/proper-lockfile": "^4.1.4", "@types/ps-tree": "^1.1.6", "@types/semver-compare": "^1.0.3", + "@types/sharp": "^0.32.0", "@types/shell-quote": "^1.7.5", "@types/stream-json": "^1.7.8", "@types/string-similarity": "^4.0.2", diff --git a/webview-ui/src/components/settings/ContextManagementSettings.tsx b/webview-ui/src/components/settings/ContextManagementSettings.tsx index 8663ea6e03..23d23764dd 100644 --- a/webview-ui/src/components/settings/ContextManagementSettings.tsx +++ b/webview-ui/src/components/settings/ContextManagementSettings.tsx @@ -35,6 +35,8 @@ type ContextManagementSettingsProps = HTMLAttributes & { enableSubfolderRules?: boolean maxImageFileSize?: number maxTotalImageSize?: number + maxImageDimension?: number + imageDownscaleQuality?: number profileThresholds?: Record includeDiagnosticMessages?: boolean maxDiagnosticMessages?: number @@ -53,6 +55,8 @@ type ContextManagementSettingsProps = HTMLAttributes & { | "enableSubfolderRules" | "maxImageFileSize" | "maxTotalImageSize" + | "maxImageDimension" + | "imageDownscaleQuality" | "profileThresholds" | "includeDiagnosticMessages" | "maxDiagnosticMessages" @@ -74,6 +78,8 @@ export const ContextManagementSettings = ({ setCachedStateField, maxImageFileSize, maxTotalImageSize, + maxImageDimension, + imageDownscaleQuality, profileThresholds = {}, includeDiagnosticMessages, maxDiagnosticMessages, @@ -308,6 +314,66 @@ export const ContextManagementSettings = ({ + +
+ {t("settings:contextManagement.maxImageDimension.label")} +
+ { + const newValue = parseInt(e.target.value, 10) + if (!isNaN(newValue) && newValue >= 0 && newValue <= 8192) { + setCachedStateField("maxImageDimension", newValue) + } + }} + onClick={(e) => e.currentTarget.select()} + data-testid="max-image-dimension-input" + /> + + {(maxImageDimension ?? 0) === 0 + ? t("settings:contextManagement.maxImageDimension.disabled") + : t("settings:contextManagement.maxImageDimension.px")} + +
+
+
+ {t("settings:contextManagement.maxImageDimension.description")} +
+ + + {(maxImageDimension ?? 0) > 0 && ( + + + {t("settings:contextManagement.imageDownscaleQuality.label")} + +
+ setCachedStateField("imageDownscaleQuality", value)} + /> + {imageDownscaleQuality ?? 85} +
+
+ {t("settings:contextManagement.imageDownscaleQuality.description")} +
+ + )} + (({ onDone, t enableSubfolderRules, maxImageFileSize, maxTotalImageSize, + maxImageDimension, + imageDownscaleQuality, customSupportPrompts, profileThresholds, alwaysAllowFollowupQuestions, @@ -404,6 +406,8 @@ const SettingsView = forwardRef(({ onDone, t enableSubfolderRules: enableSubfolderRules ?? false, maxImageFileSize: maxImageFileSize ?? 5, maxTotalImageSize: maxTotalImageSize ?? 20, + maxImageDimension: maxImageDimension ?? 0, + imageDownscaleQuality: imageDownscaleQuality ?? 85, includeDiagnosticMessages: includeDiagnosticMessages !== undefined ? includeDiagnosticMessages : true, maxDiagnosticMessages: maxDiagnosticMessages ?? 50, @@ -837,6 +841,8 @@ const SettingsView = forwardRef(({ onDone, t enableSubfolderRules={enableSubfolderRules} maxImageFileSize={maxImageFileSize} maxTotalImageSize={maxTotalImageSize} + maxImageDimension={maxImageDimension} + imageDownscaleQuality={imageDownscaleQuality} profileThresholds={profileThresholds} includeDiagnosticMessages={includeDiagnosticMessages} maxDiagnosticMessages={maxDiagnosticMessages} diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 3b2497aaee..6430a196eb 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -695,6 +695,16 @@ "mb": "MB", "description": "Maximum cumulative size limit (in MB) for all images processed in a single read_file operation. When reading multiple images, each image's size is added to the total. If including another image would exceed this limit, it will be skipped." }, + "maxImageDimension": { + "label": "Image downscale max dimension", + "px": "px", + "description": "Maximum width or height (in pixels) for images before sending to the LLM. Images exceeding this dimension are proportionally downscaled, preserving aspect ratio. Set to 0 to disable downscaling.", + "disabled": "Disabled" + }, + "imageDownscaleQuality": { + "label": "Image downscale quality", + "description": "JPEG/WebP encoding quality (1-100) used when re-encoding downscaled images. Higher values preserve more detail but produce larger files. Only applies when image downscaling is enabled." + }, "diagnostics": { "includeMessages": { "label": "Automatically include diagnostics in context",