mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-11 22:51:26 +00:00
feat: add configurable image downscaling before LLM upload
Adds a new image downscaling feature that resizes images before sending them to the LLM, reducing token usage for high-resolution screenshots. Changes: - Add sharp dependency for image processing - New settings: maxImageDimension (0=disabled) and imageDownscaleQuality (1-100) - New resize-image.ts utility using sharp for proportional downscaling - Integrate resizing into all image entry points: selectImages, readImageAsDataUrlWithBuffer, processImageFile, resolveImageMentions - Add settings UI controls in Context Management settings tab - Add English translation strings for the new settings - Downscaling disabled by default (opt-in) Closes #11912
This commit is contained in:
parent
44fd975b17
commit
d39a4e1294
16 changed files with 491 additions and 48 deletions
|
|
@ -167,6 +167,20 @@ export const globalSettingsSchema = z.object({
|
|||
maxImageFileSize: z.number().optional(),
|
||||
maxTotalImageSize: z.number().optional(),
|
||||
|
||||
/**
|
||||
* Maximum dimension (width or height) in pixels for image downscaling before LLM upload.
|
||||
* Images exceeding this dimension are proportionally resized, preserving aspect ratio.
|
||||
* Set to 0 (default) to disable downscaling.
|
||||
*/
|
||||
maxImageDimension: z.number().min(0).optional(),
|
||||
|
||||
/**
|
||||
* JPEG/WebP quality (1-100) used when re-encoding resized images.
|
||||
* Only applies when maxImageDimension is set and the image is downscaled.
|
||||
* @default 85
|
||||
*/
|
||||
imageDownscaleQuality: z.number().min(1).max(100).optional(),
|
||||
|
||||
terminalOutputPreviewSize: z.enum(["small", "medium", "large"]).optional(),
|
||||
terminalShellIntegrationTimeout: z.number().optional(),
|
||||
terminalShellIntegrationDisabled: z.boolean().optional(),
|
||||
|
|
|
|||
|
|
@ -330,6 +330,8 @@ export type ExtensionState = Pick<
|
|||
maxReadFileLine?: number // Maximum line limit for read_file tool (-1 for default)
|
||||
maxImageFileSize: number // Maximum size of image files to process in MB
|
||||
maxTotalImageSize: number // Maximum total size for all images in a single read operation in MB
|
||||
maxImageDimension?: number // Maximum dimension (width or height) in pixels for image downscaling (0 = disabled)
|
||||
imageDownscaleQuality?: number // JPEG/WebP quality (1-100) for re-encoding resized images (default 85)
|
||||
|
||||
experiments: Experiments // Map of experiment IDs to their enabled state
|
||||
|
||||
|
|
|
|||
53
pnpm-lock.yaml
generated
53
pnpm-lock.yaml
generated
|
|
@ -962,6 +962,9 @@ importers:
|
|||
serialize-error:
|
||||
specifier: ^12.0.0
|
||||
version: 12.0.0
|
||||
sharp:
|
||||
specifier: ^0.34.5
|
||||
version: 0.34.5
|
||||
shell-quote:
|
||||
specifier: ^1.8.2
|
||||
version: 1.8.3
|
||||
|
|
@ -1074,6 +1077,9 @@ importers:
|
|||
'@types/semver-compare':
|
||||
specifier: ^1.0.3
|
||||
version: 1.0.3
|
||||
'@types/sharp':
|
||||
specifier: ^0.32.0
|
||||
version: 0.32.0
|
||||
'@types/shell-quote':
|
||||
specifier: ^1.7.5
|
||||
version: 1.7.5
|
||||
|
|
@ -2027,9 +2033,6 @@ packages:
|
|||
'@emnapi/core@1.4.3':
|
||||
resolution: {integrity: sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g==}
|
||||
|
||||
'@emnapi/runtime@1.4.3':
|
||||
resolution: {integrity: sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ==}
|
||||
|
||||
'@emnapi/runtime@1.8.1':
|
||||
resolution: {integrity: sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==}
|
||||
|
||||
|
|
@ -4585,6 +4588,10 @@ packages:
|
|||
'@types/semver-compare@1.0.3':
|
||||
resolution: {integrity: sha512-mVZkB2QjXmZhh+MrtwMlJ8BqUnmbiSkpd88uOWskfwB8yitBT0tBRAKt+41VRgZD9zr9Sc+Xs02qGgvzd1Rq/Q==}
|
||||
|
||||
'@types/sharp@0.32.0':
|
||||
resolution: {integrity: sha512-OOi3kL+FZDnPhVzsfD37J88FNeZh6gQsGcLc95NbeURRGvmSjeXiDcyWzF2o3yh/gQAUn2uhh/e+CPCa5nwAxw==}
|
||||
deprecated: This is a stub types definition. sharp provides its own type definitions, so you do not need this installed.
|
||||
|
||||
'@types/shell-quote@1.7.5':
|
||||
resolution: {integrity: sha512-+UE8GAGRPbJVQDdxi16dgadcBfQ+KG2vgZhV1+3A1XmHbmwcdwhCUwIdy+d3pAGrbvgRoVSjeI9vOWyq376Yzw==}
|
||||
|
||||
|
|
@ -5130,6 +5137,7 @@ packages:
|
|||
basic-ftp@5.0.5:
|
||||
resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
deprecated: Security vulnerability fixed in 5.2.0, please upgrade
|
||||
|
||||
better-path-resolve@1.0.0:
|
||||
resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==}
|
||||
|
|
@ -5957,10 +5965,6 @@ packages:
|
|||
resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
detect-libc@2.0.4:
|
||||
resolution: {integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
detect-libc@2.1.2:
|
||||
resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -8976,6 +8980,7 @@ packages:
|
|||
prebuild-install@7.1.3:
|
||||
resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==}
|
||||
engines: {node: '>=10'}
|
||||
deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available.
|
||||
hasBin: true
|
||||
|
||||
prelude-ls@1.2.1:
|
||||
|
|
@ -12149,11 +12154,6 @@ snapshots:
|
|||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@emnapi/runtime@1.4.3':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@emnapi/runtime@1.8.1':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
|
@ -12391,8 +12391,7 @@ snapshots:
|
|||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@img/colour@1.0.0':
|
||||
optional: true
|
||||
'@img/colour@1.0.0': {}
|
||||
|
||||
'@img/sharp-darwin-arm64@0.34.5':
|
||||
optionalDependencies:
|
||||
|
|
@ -12733,14 +12732,14 @@ snapshots:
|
|||
'@napi-rs/wasm-runtime@0.2.10':
|
||||
dependencies:
|
||||
'@emnapi/core': 1.4.3
|
||||
'@emnapi/runtime': 1.4.3
|
||||
'@emnapi/runtime': 1.8.1
|
||||
'@tybys/wasm-util': 0.9.0
|
||||
optional: true
|
||||
|
||||
'@napi-rs/wasm-runtime@0.2.11':
|
||||
dependencies:
|
||||
'@emnapi/core': 1.4.3
|
||||
'@emnapi/runtime': 1.4.3
|
||||
'@emnapi/runtime': 1.8.1
|
||||
'@tybys/wasm-util': 0.9.0
|
||||
optional: true
|
||||
|
||||
|
|
@ -14319,7 +14318,7 @@ snapshots:
|
|||
|
||||
'@tailwindcss/oxide@4.1.6':
|
||||
dependencies:
|
||||
detect-libc: 2.0.4
|
||||
detect-libc: 2.1.2
|
||||
tar: 7.4.3
|
||||
optionalDependencies:
|
||||
'@tailwindcss/oxide-android-arm64': 4.1.6
|
||||
|
|
@ -14337,7 +14336,7 @@ snapshots:
|
|||
|
||||
'@tailwindcss/oxide@4.1.8':
|
||||
dependencies:
|
||||
detect-libc: 2.0.4
|
||||
detect-libc: 2.1.2
|
||||
tar: 7.4.3
|
||||
optionalDependencies:
|
||||
'@tailwindcss/oxide-android-arm64': 4.1.8
|
||||
|
|
@ -14720,6 +14719,10 @@ snapshots:
|
|||
|
||||
'@types/semver-compare@1.0.3': {}
|
||||
|
||||
'@types/sharp@0.32.0':
|
||||
dependencies:
|
||||
sharp: 0.34.5
|
||||
|
||||
'@types/shell-quote@1.7.5': {}
|
||||
|
||||
'@types/stack-utils@2.0.3': {}
|
||||
|
|
@ -14974,7 +14977,7 @@ snapshots:
|
|||
sirv: 3.0.1
|
||||
tinyglobby: 0.2.14
|
||||
tinyrainbow: 2.0.0
|
||||
vitest: 3.2.4(@types/debug@4.1.12)(@types/node@24.2.1)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)
|
||||
vitest: 3.2.4(@types/debug@4.1.12)(@types/node@20.17.50)(@vitest/ui@3.2.4)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)
|
||||
|
||||
'@vitest/utils@3.2.4':
|
||||
dependencies:
|
||||
|
|
@ -16253,10 +16256,7 @@ snapshots:
|
|||
detect-libc@2.0.2:
|
||||
optional: true
|
||||
|
||||
detect-libc@2.0.4: {}
|
||||
|
||||
detect-libc@2.1.2:
|
||||
optional: true
|
||||
detect-libc@2.1.2: {}
|
||||
|
||||
detect-node-es@1.1.0: {}
|
||||
|
||||
|
|
@ -18333,7 +18333,7 @@ snapshots:
|
|||
|
||||
lightningcss@1.29.2:
|
||||
dependencies:
|
||||
detect-libc: 2.0.4
|
||||
detect-libc: 2.1.2
|
||||
optionalDependencies:
|
||||
lightningcss-darwin-arm64: 1.29.2
|
||||
lightningcss-darwin-x64: 1.29.2
|
||||
|
|
@ -18348,7 +18348,7 @@ snapshots:
|
|||
|
||||
lightningcss@1.30.1:
|
||||
dependencies:
|
||||
detect-libc: 2.0.4
|
||||
detect-libc: 2.1.2
|
||||
optionalDependencies:
|
||||
lightningcss-darwin-arm64: 1.30.1
|
||||
lightningcss-darwin-x64: 1.30.1
|
||||
|
|
@ -19761,7 +19761,7 @@ snapshots:
|
|||
|
||||
prebuild-install@7.1.3:
|
||||
dependencies:
|
||||
detect-libc: 2.0.4
|
||||
detect-libc: 2.1.2
|
||||
expand-template: 2.0.3
|
||||
github-from-package: 0.0.0
|
||||
minimist: 1.2.8
|
||||
|
|
@ -20639,7 +20639,6 @@ snapshots:
|
|||
'@img/sharp-win32-arm64': 0.34.5
|
||||
'@img/sharp-win32-ia32': 0.34.5
|
||||
'@img/sharp-win32-x64': 0.34.5
|
||||
optional: true
|
||||
|
||||
shebang-command@2.0.0:
|
||||
dependencies:
|
||||
|
|
|
|||
|
|
@ -41,7 +41,10 @@ describe("resolveImageMentions", () => {
|
|||
})
|
||||
|
||||
expect(mockValidateImage).toHaveBeenCalled()
|
||||
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"))
|
||||
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"), {
|
||||
maxDimension: undefined,
|
||||
quality: undefined,
|
||||
})
|
||||
expect(result.text).toBe("Please look at @/assets/cat.png")
|
||||
expect(result.images).toEqual([dataUrl])
|
||||
})
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ export interface ResolveImageMentionsOptions {
|
|||
maxImageFileSize?: number
|
||||
/** Maximum total size of all images in MB. Defaults to 20MB. */
|
||||
maxTotalImageSize?: number
|
||||
/** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */
|
||||
maxImageDimension?: number
|
||||
/** JPEG/WebP quality (1-100) for re-encoding resized images. */
|
||||
imageDownscaleQuality?: number
|
||||
}
|
||||
|
||||
export interface ResolveImageMentionsResult {
|
||||
|
|
@ -65,6 +69,8 @@ export async function resolveImageMentions({
|
|||
supportsImages = true,
|
||||
maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
|
||||
maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
}: ResolveImageMentionsOptions): Promise<ResolveImageMentionsResult> {
|
||||
const existingImages = Array.isArray(images) ? images : []
|
||||
if (existingImages.length >= MAX_IMAGES_PER_MESSAGE) {
|
||||
|
|
@ -127,7 +133,10 @@ export async function resolveImageMentions({
|
|||
continue
|
||||
}
|
||||
|
||||
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath)
|
||||
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath, {
|
||||
maxDimension: maxImageDimension,
|
||||
quality: imageDownscaleQuality,
|
||||
})
|
||||
newImages.push(dataUrl)
|
||||
|
||||
// Track memory usage
|
||||
|
|
|
|||
|
|
@ -171,6 +171,8 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
const {
|
||||
maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
|
||||
maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
} = state ?? {}
|
||||
|
||||
for (const fileResult of fileResults) {
|
||||
|
|
@ -207,6 +209,8 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
maxTotalImageSize,
|
||||
imageMemoryTracker,
|
||||
updateFileResult,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
|
@ -341,6 +345,8 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
maxTotalImageSize: number,
|
||||
imageMemoryTracker: ImageMemoryTracker,
|
||||
updateFileResult: (path: string, updates: Partial<FileResult>) => void,
|
||||
maxImageDimension?: number,
|
||||
imageDownscaleQuality?: number,
|
||||
): Promise<void> {
|
||||
const fileExtension = path.extname(relPath).toLowerCase()
|
||||
const supportedBinaryFormats = getSupportedBinaryFormats()
|
||||
|
|
@ -364,7 +370,10 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
return
|
||||
}
|
||||
|
||||
const imageResult = await processImageFile(fullPath)
|
||||
const imageResult = await processImageFile(fullPath, {
|
||||
maxDimension: maxImageDimension,
|
||||
quality: imageDownscaleQuality,
|
||||
})
|
||||
imageMemoryTracker.addMemoryUsage(imageResult.sizeInMB)
|
||||
await task.fileContextTracker.trackFileContext(relPath, "read_tool" as RecordSource)
|
||||
|
||||
|
|
@ -744,6 +753,8 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
const {
|
||||
maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
|
||||
maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
|
||||
maxImageDimension: legacyMaxDim,
|
||||
imageDownscaleQuality: legacyQuality,
|
||||
} = state ?? {}
|
||||
const validation = await validateImageForProcessing(
|
||||
fullPath,
|
||||
|
|
@ -756,7 +767,10 @@ export class ReadFileTool extends BaseTool<"read_file"> {
|
|||
results.push(`File: ${relPath}\nNotice: ${validation.notice ?? "Image validation failed"}`)
|
||||
continue
|
||||
}
|
||||
const imageResult = await processImageFile(fullPath)
|
||||
const imageResult = await processImageFile(fullPath, {
|
||||
maxDimension: legacyMaxDim,
|
||||
quality: legacyQuality,
|
||||
})
|
||||
if (imageResult) {
|
||||
results.push(`File: ${relPath}\n[Image file - content processed for vision model]`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import path from "path"
|
|||
import * as fs from "fs/promises"
|
||||
import { t } from "../../../i18n"
|
||||
import prettyBytes from "pretty-bytes"
|
||||
import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "../../../integrations/misc/resize-image"
|
||||
|
||||
/**
|
||||
* Default maximum allowed image file size in bytes (5MB)
|
||||
|
|
@ -69,15 +70,37 @@ export interface ImageProcessingResult {
|
|||
notice: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an image file and returns both the data URL and buffer
|
||||
*/
|
||||
export async function readImageAsDataUrlWithBuffer(filePath: string): Promise<{ dataUrl: string; buffer: Buffer }> {
|
||||
const fileBuffer = await fs.readFile(filePath)
|
||||
const base64 = fileBuffer.toString("base64")
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
export interface ReadImageOptions {
|
||||
/** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */
|
||||
maxDimension?: number
|
||||
/** JPEG/WebP quality (1-100) for re-encoding resized images. */
|
||||
quality?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an image file and returns both the data URL and buffer.
|
||||
* Optionally downscales the image if maxDimension is set.
|
||||
*/
|
||||
export async function readImageAsDataUrlWithBuffer(
|
||||
filePath: string,
|
||||
options?: ReadImageOptions,
|
||||
): Promise<{ dataUrl: string; buffer: Buffer }> {
|
||||
let fileBuffer = await fs.readFile(filePath)
|
||||
const ext = path.extname(filePath).toLowerCase()
|
||||
const mimeType = IMAGE_MIME_TYPES[ext] || "image/png"
|
||||
|
||||
// Downscale if configured
|
||||
if (options?.maxDimension && options.maxDimension > 0) {
|
||||
const resizeResult = await maybeResizeImage({
|
||||
buffer: fileBuffer,
|
||||
mimeType,
|
||||
maxDimension: options.maxDimension,
|
||||
quality: options.quality ?? DEFAULT_IMAGE_DOWNSCALE_QUALITY,
|
||||
})
|
||||
fileBuffer = resizeResult.buffer
|
||||
}
|
||||
|
||||
const base64 = fileBuffer.toString("base64")
|
||||
const dataUrl = `data:${mimeType};base64,${base64}`
|
||||
|
||||
return { dataUrl, buffer: fileBuffer }
|
||||
|
|
@ -145,11 +168,12 @@ export async function validateImageForProcessing(
|
|||
}
|
||||
|
||||
/**
|
||||
* Processes an image file and returns the result
|
||||
* Processes an image file and returns the result.
|
||||
* Optionally downscales the image if resize options are provided.
|
||||
*/
|
||||
export async function processImageFile(fullPath: string): Promise<ImageProcessingResult> {
|
||||
export async function processImageFile(fullPath: string, options?: ReadImageOptions): Promise<ImageProcessingResult> {
|
||||
const imageStats = await fs.stat(fullPath)
|
||||
const { dataUrl, buffer } = await readImageAsDataUrlWithBuffer(fullPath)
|
||||
const { dataUrl, buffer } = await readImageAsDataUrlWithBuffer(fullPath, options)
|
||||
const imageSizeInKB = Math.round(imageStats.size / 1024)
|
||||
const imageSizeInMB = imageStats.size / (1024 * 1024)
|
||||
const noticeText = t("tools:readFile.imageWithSize", { size: imageSizeInKB })
|
||||
|
|
|
|||
|
|
@ -2182,6 +2182,8 @@ export class ClineProvider
|
|||
language,
|
||||
maxImageFileSize,
|
||||
maxTotalImageSize,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
historyPreviewCollapsed,
|
||||
reasoningBlockCollapsed,
|
||||
enterBehavior,
|
||||
|
|
@ -2307,6 +2309,8 @@ export class ClineProvider
|
|||
renderContext: this.renderContext,
|
||||
maxImageFileSize: maxImageFileSize ?? 5,
|
||||
maxTotalImageSize: maxTotalImageSize ?? 20,
|
||||
maxImageDimension: maxImageDimension ?? 0,
|
||||
imageDownscaleQuality: imageDownscaleQuality ?? 85,
|
||||
settingsImportedAt: this.settingsImportedAt,
|
||||
historyPreviewCollapsed: historyPreviewCollapsed ?? false,
|
||||
reasoningBlockCollapsed: reasoningBlockCollapsed ?? true,
|
||||
|
|
@ -2531,6 +2535,8 @@ export class ClineProvider
|
|||
enableSubfolderRules: stateValues.enableSubfolderRules ?? false,
|
||||
maxImageFileSize: stateValues.maxImageFileSize ?? 5,
|
||||
maxTotalImageSize: stateValues.maxTotalImageSize ?? 20,
|
||||
maxImageDimension: stateValues.maxImageDimension ?? 0,
|
||||
imageDownscaleQuality: stateValues.imageDownscaleQuality ?? 85,
|
||||
historyPreviewCollapsed: stateValues.historyPreviewCollapsed ?? false,
|
||||
reasoningBlockCollapsed: stateValues.reasoningBlockCollapsed ?? true,
|
||||
enterBehavior: stateValues.enterBehavior ?? "send",
|
||||
|
|
|
|||
|
|
@ -185,6 +185,8 @@ export const webviewMessageHandler = async (
|
|||
rooIgnoreController: currentTask?.rooIgnoreController,
|
||||
maxImageFileSize: state.maxImageFileSize,
|
||||
maxTotalImageSize: state.maxTotalImageSize,
|
||||
maxImageDimension: state.maxImageDimension,
|
||||
imageDownscaleQuality: state.imageDownscaleQuality,
|
||||
})
|
||||
return resolved
|
||||
}
|
||||
|
|
@ -766,8 +768,12 @@ export const webviewMessageHandler = async (
|
|||
await updateGlobalState("lastShownAnnouncementId", provider.latestAnnouncementId)
|
||||
await provider.postStateToWebview()
|
||||
break
|
||||
case "selectImages":
|
||||
const images = await selectImages()
|
||||
case "selectImages": {
|
||||
const selectState = await provider.getState()
|
||||
const images = await selectImages({
|
||||
maxDimension: selectState.maxImageDimension,
|
||||
quality: selectState.imageDownscaleQuality,
|
||||
})
|
||||
await provider.postMessageToWebview({
|
||||
type: "selectedImages",
|
||||
images,
|
||||
|
|
@ -775,6 +781,7 @@ export const webviewMessageHandler = async (
|
|||
messageTs: message.messageTs,
|
||||
})
|
||||
break
|
||||
}
|
||||
case "exportCurrentTask":
|
||||
const currentTaskId = provider.getCurrentTask()?.taskId
|
||||
if (currentTaskId) {
|
||||
|
|
|
|||
162
src/integrations/misc/__tests__/resize-image.spec.ts
Normal file
162
src/integrations/misc/__tests__/resize-image.spec.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "../resize-image"
|
||||
import sharp from "sharp"
|
||||
|
||||
describe("maybeResizeImage", () => {
|
||||
/**
|
||||
* Helper to create a test image buffer with specified dimensions.
|
||||
*/
|
||||
async function createTestImage(
|
||||
width: number,
|
||||
height: number,
|
||||
format: "png" | "jpeg" | "webp" = "png",
|
||||
): Promise<Buffer> {
|
||||
const channels = 3
|
||||
const rawData = Buffer.alloc(width * height * channels, 128)
|
||||
let pipeline = sharp(rawData, { raw: { width, height, channels } })
|
||||
|
||||
if (format === "png") {
|
||||
pipeline = pipeline.png()
|
||||
} else if (format === "jpeg") {
|
||||
pipeline = pipeline.jpeg()
|
||||
} else if (format === "webp") {
|
||||
pipeline = pipeline.webp()
|
||||
}
|
||||
|
||||
return pipeline.toBuffer()
|
||||
}
|
||||
|
||||
it("should return original buffer when maxDimension is 0", async () => {
|
||||
const buffer = await createTestImage(100, 100)
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
maxDimension: 0,
|
||||
})
|
||||
expect(result.wasResized).toBe(false)
|
||||
expect(result.buffer).toBe(buffer) // same reference
|
||||
})
|
||||
|
||||
it("should return original buffer when maxDimension is undefined", async () => {
|
||||
const buffer = await createTestImage(100, 100)
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
})
|
||||
expect(result.wasResized).toBe(false)
|
||||
expect(result.buffer).toBe(buffer)
|
||||
})
|
||||
|
||||
it("should not resize when image is smaller than maxDimension", async () => {
|
||||
const buffer = await createTestImage(200, 100)
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
maxDimension: 300,
|
||||
})
|
||||
expect(result.wasResized).toBe(false)
|
||||
expect(result.buffer).toBe(buffer)
|
||||
})
|
||||
|
||||
it("should downscale a wide image exceeding maxDimension", async () => {
|
||||
const buffer = await createTestImage(2000, 1000, "png")
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
maxDimension: 500,
|
||||
})
|
||||
expect(result.wasResized).toBe(true)
|
||||
|
||||
const metadata = await sharp(result.buffer).metadata()
|
||||
expect(metadata.width).toBeLessThanOrEqual(500)
|
||||
expect(metadata.height).toBeLessThanOrEqual(500)
|
||||
// Check aspect ratio is roughly preserved (2:1)
|
||||
expect(metadata.width! / metadata.height!).toBeCloseTo(2, 0)
|
||||
})
|
||||
|
||||
it("should downscale a tall image exceeding maxDimension", async () => {
|
||||
const buffer = await createTestImage(500, 2000, "jpeg")
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/jpeg",
|
||||
maxDimension: 1000,
|
||||
})
|
||||
expect(result.wasResized).toBe(true)
|
||||
|
||||
const metadata = await sharp(result.buffer).metadata()
|
||||
expect(metadata.width).toBeLessThanOrEqual(1000)
|
||||
expect(metadata.height).toBeLessThanOrEqual(1000)
|
||||
// Check aspect ratio is roughly preserved (1:4)
|
||||
expect(metadata.height! / metadata.width!).toBeCloseTo(4, 0)
|
||||
})
|
||||
|
||||
it("should handle webp format", async () => {
|
||||
const buffer = await createTestImage(1500, 1500, "webp")
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/webp",
|
||||
maxDimension: 800,
|
||||
})
|
||||
expect(result.wasResized).toBe(true)
|
||||
|
||||
const metadata = await sharp(result.buffer).metadata()
|
||||
expect(metadata.format).toBe("webp")
|
||||
expect(metadata.width).toBeLessThanOrEqual(800)
|
||||
expect(metadata.height).toBeLessThanOrEqual(800)
|
||||
})
|
||||
|
||||
it("should return original buffer for unsupported mime types", async () => {
|
||||
const buffer = await createTestImage(2000, 2000)
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/gif",
|
||||
maxDimension: 500,
|
||||
})
|
||||
expect(result.wasResized).toBe(false)
|
||||
expect(result.buffer).toBe(buffer)
|
||||
})
|
||||
|
||||
it("should use default quality when not specified", async () => {
|
||||
expect(DEFAULT_IMAGE_DOWNSCALE_QUALITY).toBe(85)
|
||||
})
|
||||
|
||||
it("should accept custom quality setting for jpeg without error", async () => {
|
||||
const buffer = await createTestImage(2000, 2000, "jpeg")
|
||||
|
||||
const resultHighQ = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/jpeg",
|
||||
maxDimension: 500,
|
||||
quality: 95,
|
||||
})
|
||||
|
||||
const resultLowQ = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/jpeg",
|
||||
maxDimension: 500,
|
||||
quality: 10,
|
||||
})
|
||||
|
||||
// Both should be resized successfully
|
||||
expect(resultHighQ.wasResized).toBe(true)
|
||||
expect(resultLowQ.wasResized).toBe(true)
|
||||
|
||||
// Both should produce valid image buffers
|
||||
const metaHigh = await sharp(resultHighQ.buffer).metadata()
|
||||
const metaLow = await sharp(resultLowQ.buffer).metadata()
|
||||
expect(metaHigh.format).toBe("jpeg")
|
||||
expect(metaLow.format).toBe("jpeg")
|
||||
expect(metaHigh.width).toBeLessThanOrEqual(500)
|
||||
expect(metaLow.width).toBeLessThanOrEqual(500)
|
||||
})
|
||||
|
||||
it("should not upscale small images", async () => {
|
||||
const buffer = await createTestImage(100, 50, "png")
|
||||
const result = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType: "image/png",
|
||||
maxDimension: 500,
|
||||
})
|
||||
expect(result.wasResized).toBe(false)
|
||||
expect(result.buffer).toBe(buffer)
|
||||
})
|
||||
})
|
||||
|
|
@ -1,9 +1,17 @@
|
|||
import * as vscode from "vscode"
|
||||
import fs from "fs/promises"
|
||||
import * as path from "path"
|
||||
import { maybeResizeImage, DEFAULT_IMAGE_DOWNSCALE_QUALITY } from "./resize-image"
|
||||
|
||||
export async function selectImages(): Promise<string[]> {
|
||||
const options: vscode.OpenDialogOptions = {
|
||||
export interface SelectImagesOptions {
|
||||
/** Maximum dimension (width or height) in pixels for downscaling. 0 = disabled. */
|
||||
maxDimension?: number
|
||||
/** JPEG/WebP quality (1-100) for re-encoding resized images. */
|
||||
quality?: number
|
||||
}
|
||||
|
||||
export async function selectImages(options?: SelectImagesOptions): Promise<string[]> {
|
||||
const dialogOptions: vscode.OpenDialogOptions = {
|
||||
canSelectMany: true,
|
||||
openLabel: "Select",
|
||||
filters: {
|
||||
|
|
@ -11,7 +19,7 @@ export async function selectImages(): Promise<string[]> {
|
|||
},
|
||||
}
|
||||
|
||||
const fileUris = await vscode.window.showOpenDialog(options)
|
||||
const fileUris = await vscode.window.showOpenDialog(dialogOptions)
|
||||
|
||||
if (!fileUris || fileUris.length === 0) {
|
||||
return []
|
||||
|
|
@ -20,9 +28,21 @@ export async function selectImages(): Promise<string[]> {
|
|||
return await Promise.all(
|
||||
fileUris.map(async (uri) => {
|
||||
const imagePath = uri.fsPath
|
||||
const buffer = await fs.readFile(imagePath)
|
||||
const base64 = buffer.toString("base64")
|
||||
let buffer = await fs.readFile(imagePath)
|
||||
const mimeType = getMimeType(imagePath)
|
||||
|
||||
// Downscale if configured
|
||||
if (options?.maxDimension && options.maxDimension > 0) {
|
||||
const resizeResult = await maybeResizeImage({
|
||||
buffer,
|
||||
mimeType,
|
||||
maxDimension: options.maxDimension,
|
||||
quality: options.quality ?? DEFAULT_IMAGE_DOWNSCALE_QUALITY,
|
||||
})
|
||||
buffer = resizeResult.buffer
|
||||
}
|
||||
|
||||
const base64 = buffer.toString("base64")
|
||||
const dataUrl = `data:${mimeType};base64,${base64}`
|
||||
return dataUrl
|
||||
}),
|
||||
|
|
|
|||
99
src/integrations/misc/resize-image.ts
Normal file
99
src/integrations/misc/resize-image.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import sharp from "sharp"
|
||||
|
||||
/**
|
||||
* Default image downscale quality (JPEG/WebP) when re-encoding resized images.
|
||||
*/
|
||||
export const DEFAULT_IMAGE_DOWNSCALE_QUALITY = 85
|
||||
|
||||
/**
|
||||
* Mime type to sharp output format mapping.
|
||||
*/
|
||||
const MIME_TO_FORMAT: Record<string, keyof sharp.FormatEnum> = {
|
||||
"image/png": "png",
|
||||
"image/jpeg": "jpeg",
|
||||
"image/webp": "webp",
|
||||
}
|
||||
|
||||
export interface ResizeImageOptions {
|
||||
/** The image buffer to potentially resize. */
|
||||
buffer: Buffer
|
||||
/** The MIME type of the image (e.g. "image/png"). */
|
||||
mimeType: string
|
||||
/** Maximum dimension (width or height) in pixels. 0 or undefined means no resizing. */
|
||||
maxDimension?: number
|
||||
/** JPEG/WebP quality (1-100) for re-encoding. Defaults to 85. */
|
||||
quality?: number
|
||||
}
|
||||
|
||||
export interface ResizeImageResult {
|
||||
/** The (possibly resized) image buffer. */
|
||||
buffer: Buffer
|
||||
/** Whether the image was actually resized. */
|
||||
wasResized: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Conditionally downscales an image buffer if either dimension exceeds `maxDimension`.
|
||||
* Preserves aspect ratio. Returns the original buffer unchanged if no resizing is needed
|
||||
* or if the format is unsupported for resizing.
|
||||
*/
|
||||
export async function maybeResizeImage({
|
||||
buffer,
|
||||
mimeType,
|
||||
maxDimension,
|
||||
quality = DEFAULT_IMAGE_DOWNSCALE_QUALITY,
|
||||
}: ResizeImageOptions): Promise<ResizeImageResult> {
|
||||
// If downscaling is disabled or dimension is 0/undefined, return as-is
|
||||
if (!maxDimension || maxDimension <= 0) {
|
||||
return { buffer, wasResized: false }
|
||||
}
|
||||
|
||||
// Only resize formats we can handle
|
||||
const format = MIME_TO_FORMAT[mimeType]
|
||||
if (!format) {
|
||||
return { buffer, wasResized: false }
|
||||
}
|
||||
|
||||
const image = sharp(buffer)
|
||||
const metadata = await image.metadata()
|
||||
|
||||
if (!metadata.width || !metadata.height) {
|
||||
return { buffer, wasResized: false }
|
||||
}
|
||||
|
||||
// Only downscale -- never upscale
|
||||
if (metadata.width <= maxDimension && metadata.height <= maxDimension) {
|
||||
return { buffer, wasResized: false }
|
||||
}
|
||||
|
||||
// Calculate new dimensions preserving aspect ratio
|
||||
const aspectRatio = metadata.width / metadata.height
|
||||
let newWidth: number
|
||||
let newHeight: number
|
||||
|
||||
if (metadata.width >= metadata.height) {
|
||||
newWidth = maxDimension
|
||||
newHeight = Math.round(maxDimension / aspectRatio)
|
||||
} else {
|
||||
newHeight = maxDimension
|
||||
newWidth = Math.round(maxDimension * aspectRatio)
|
||||
}
|
||||
|
||||
// Perform the resize
|
||||
let pipeline = image.resize(newWidth, newHeight, {
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
|
||||
// Re-encode in the original format with quality setting where applicable
|
||||
if (format === "jpeg") {
|
||||
pipeline = pipeline.jpeg({ quality })
|
||||
} else if (format === "webp") {
|
||||
pipeline = pipeline.webp({ quality })
|
||||
} else if (format === "png") {
|
||||
pipeline = pipeline.png()
|
||||
}
|
||||
|
||||
const resizedBuffer = await pipeline.toBuffer()
|
||||
return { buffer: resizedBuffer, wasResized: true }
|
||||
}
|
||||
|
|
@ -519,6 +519,7 @@
|
|||
"say": "^0.16.0",
|
||||
"semver-compare": "^1.0.0",
|
||||
"serialize-error": "^12.0.0",
|
||||
"sharp": "^0.34.5",
|
||||
"shell-quote": "^1.8.2",
|
||||
"simple-git": "^3.27.0",
|
||||
"sound-play": "^1.1.0",
|
||||
|
|
@ -558,6 +559,7 @@
|
|||
"@types/proper-lockfile": "^4.1.4",
|
||||
"@types/ps-tree": "^1.1.6",
|
||||
"@types/semver-compare": "^1.0.3",
|
||||
"@types/sharp": "^0.32.0",
|
||||
"@types/shell-quote": "^1.7.5",
|
||||
"@types/stream-json": "^1.7.8",
|
||||
"@types/string-similarity": "^4.0.2",
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ type ContextManagementSettingsProps = HTMLAttributes<HTMLDivElement> & {
|
|||
enableSubfolderRules?: boolean
|
||||
maxImageFileSize?: number
|
||||
maxTotalImageSize?: number
|
||||
maxImageDimension?: number
|
||||
imageDownscaleQuality?: number
|
||||
profileThresholds?: Record<string, number>
|
||||
includeDiagnosticMessages?: boolean
|
||||
maxDiagnosticMessages?: number
|
||||
|
|
@ -53,6 +55,8 @@ type ContextManagementSettingsProps = HTMLAttributes<HTMLDivElement> & {
|
|||
| "enableSubfolderRules"
|
||||
| "maxImageFileSize"
|
||||
| "maxTotalImageSize"
|
||||
| "maxImageDimension"
|
||||
| "imageDownscaleQuality"
|
||||
| "profileThresholds"
|
||||
| "includeDiagnosticMessages"
|
||||
| "maxDiagnosticMessages"
|
||||
|
|
@ -74,6 +78,8 @@ export const ContextManagementSettings = ({
|
|||
setCachedStateField,
|
||||
maxImageFileSize,
|
||||
maxTotalImageSize,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
profileThresholds = {},
|
||||
includeDiagnosticMessages,
|
||||
maxDiagnosticMessages,
|
||||
|
|
@ -308,6 +314,66 @@ export const ContextManagementSettings = ({
|
|||
</div>
|
||||
</SearchableSetting>
|
||||
|
||||
<SearchableSetting
|
||||
settingId="context-max-image-dimension"
|
||||
section="contextManagement"
|
||||
label={t("settings:contextManagement.maxImageDimension.label")}>
|
||||
<div className="flex flex-col gap-2">
|
||||
<span className="font-medium">{t("settings:contextManagement.maxImageDimension.label")}</span>
|
||||
<div className="flex items-center gap-4">
|
||||
<Input
|
||||
type="number"
|
||||
pattern="[0-9]*"
|
||||
className="w-24 bg-vscode-input-background text-vscode-input-foreground border border-vscode-input-border px-2 py-1 rounded text-right [appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none"
|
||||
value={maxImageDimension ?? 0}
|
||||
min={0}
|
||||
max={8192}
|
||||
step={256}
|
||||
onChange={(e) => {
|
||||
const newValue = parseInt(e.target.value, 10)
|
||||
if (!isNaN(newValue) && newValue >= 0 && newValue <= 8192) {
|
||||
setCachedStateField("maxImageDimension", newValue)
|
||||
}
|
||||
}}
|
||||
onClick={(e) => e.currentTarget.select()}
|
||||
data-testid="max-image-dimension-input"
|
||||
/>
|
||||
<span>
|
||||
{(maxImageDimension ?? 0) === 0
|
||||
? t("settings:contextManagement.maxImageDimension.disabled")
|
||||
: t("settings:contextManagement.maxImageDimension.px")}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="text-vscode-descriptionForeground text-sm mt-2">
|
||||
{t("settings:contextManagement.maxImageDimension.description")}
|
||||
</div>
|
||||
</SearchableSetting>
|
||||
|
||||
{(maxImageDimension ?? 0) > 0 && (
|
||||
<SearchableSetting
|
||||
settingId="context-image-downscale-quality"
|
||||
section="contextManagement"
|
||||
label={t("settings:contextManagement.imageDownscaleQuality.label")}>
|
||||
<span className="block font-medium mb-1">
|
||||
{t("settings:contextManagement.imageDownscaleQuality.label")}
|
||||
</span>
|
||||
<div className="flex items-center gap-2">
|
||||
<Slider
|
||||
min={1}
|
||||
max={100}
|
||||
step={1}
|
||||
value={[imageDownscaleQuality ?? 85]}
|
||||
onValueChange={([value]) => setCachedStateField("imageDownscaleQuality", value)}
|
||||
/>
|
||||
<span className="w-10">{imageDownscaleQuality ?? 85}</span>
|
||||
</div>
|
||||
<div className="text-vscode-descriptionForeground text-sm mt-1">
|
||||
{t("settings:contextManagement.imageDownscaleQuality.description")}
|
||||
</div>
|
||||
</SearchableSetting>
|
||||
)}
|
||||
|
||||
<SearchableSetting
|
||||
settingId="context-include-diagnostic-messages"
|
||||
section="contextManagement"
|
||||
|
|
|
|||
|
|
@ -188,6 +188,8 @@ const SettingsView = forwardRef<SettingsViewRef, SettingsViewProps>(({ onDone, t
|
|||
enableSubfolderRules,
|
||||
maxImageFileSize,
|
||||
maxTotalImageSize,
|
||||
maxImageDimension,
|
||||
imageDownscaleQuality,
|
||||
customSupportPrompts,
|
||||
profileThresholds,
|
||||
alwaysAllowFollowupQuestions,
|
||||
|
|
@ -404,6 +406,8 @@ const SettingsView = forwardRef<SettingsViewRef, SettingsViewProps>(({ onDone, t
|
|||
enableSubfolderRules: enableSubfolderRules ?? false,
|
||||
maxImageFileSize: maxImageFileSize ?? 5,
|
||||
maxTotalImageSize: maxTotalImageSize ?? 20,
|
||||
maxImageDimension: maxImageDimension ?? 0,
|
||||
imageDownscaleQuality: imageDownscaleQuality ?? 85,
|
||||
includeDiagnosticMessages:
|
||||
includeDiagnosticMessages !== undefined ? includeDiagnosticMessages : true,
|
||||
maxDiagnosticMessages: maxDiagnosticMessages ?? 50,
|
||||
|
|
@ -837,6 +841,8 @@ const SettingsView = forwardRef<SettingsViewRef, SettingsViewProps>(({ onDone, t
|
|||
enableSubfolderRules={enableSubfolderRules}
|
||||
maxImageFileSize={maxImageFileSize}
|
||||
maxTotalImageSize={maxTotalImageSize}
|
||||
maxImageDimension={maxImageDimension}
|
||||
imageDownscaleQuality={imageDownscaleQuality}
|
||||
profileThresholds={profileThresholds}
|
||||
includeDiagnosticMessages={includeDiagnosticMessages}
|
||||
maxDiagnosticMessages={maxDiagnosticMessages}
|
||||
|
|
|
|||
|
|
@ -695,6 +695,16 @@
|
|||
"mb": "MB",
|
||||
"description": "Maximum cumulative size limit (in MB) for all images processed in a single read_file operation. When reading multiple images, each image's size is added to the total. If including another image would exceed this limit, it will be skipped."
|
||||
},
|
||||
"maxImageDimension": {
|
||||
"label": "Image downscale max dimension",
|
||||
"px": "px",
|
||||
"description": "Maximum width or height (in pixels) for images before sending to the LLM. Images exceeding this dimension are proportionally downscaled, preserving aspect ratio. Set to 0 to disable downscaling.",
|
||||
"disabled": "Disabled"
|
||||
},
|
||||
"imageDownscaleQuality": {
|
||||
"label": "Image downscale quality",
|
||||
"description": "JPEG/WebP encoding quality (1-100) used when re-encoding downscaled images. Higher values preserve more detail but produce larger files. Only applies when image downscaling is enabled."
|
||||
},
|
||||
"diagnostics": {
|
||||
"includeMessages": {
|
||||
"label": "Automatically include diagnostics in context",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue