From ca26779b2b1cd1cacae24e407a7ae718a8783d14 Mon Sep 17 00:00:00 2001 From: Roo Code Date: Wed, 30 Jul 2025 17:37:13 +0000 Subject: [PATCH] feat: limit images to 20 for AWS Bedrock browser tool usage - Add TOO_MANY_IMAGES error type to Bedrock error handling - Create image limiting utility that keeps most recent 20 images - Replace older images with descriptive text placeholders - Integrate image limiting into Bedrock message conversion - Add comprehensive tests for image limiting functionality Fixes #6348 --- src/api/providers/bedrock.ts | 18 +- .../__tests__/image-limiting.spec.ts | 378 ++++++++++++++++++ src/api/transform/image-limiting.ts | 128 ++++++ 3 files changed, 522 insertions(+), 2 deletions(-) create mode 100644 src/api/transform/__tests__/image-limiting.spec.ts create mode 100644 src/api/transform/image-limiting.ts diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index 76e502e6c7..856ade7915 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -30,6 +30,7 @@ import { MultiPointStrategy } from "../transform/cache-strategy/multi-point-stra import { ModelInfo as CacheModelInfo } from "../transform/cache-strategy/types" import { convertToBedrockConverseMessages as sharedConverter } from "../transform/bedrock-converse-format" import { getModelParams } from "../transform/model-params" +import { applyBedrockImageLimiting } from "../transform/image-limiting" import { shouldUseReasoningBudget } from "../../shared/api" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" @@ -706,8 +707,11 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH modelInfo?: any, conversationId?: string, // Optional conversation ID to track cache points across messages ): { system: SystemContentBlock[]; messages: Message[] } { + // Apply image limiting before conversion to prevent "too many images" errors + const limitedMessages = applyBedrockImageLimiting(anthropicMessages as Anthropic.Messages.MessageParam[]) + // First convert messages using shared converter for proper image handling - const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[]) + const convertedMessages = sharedConverter(limitedMessages) // If prompt caching is disabled, return the converted messages directly if (!usePromptCache) { @@ -737,7 +741,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH const config = { modelInfo: cacheModelInfo, systemPrompt: systemMessage, - messages: anthropicMessages as Anthropic.Messages.MessageParam[], + messages: limitedMessages, usePromptCache, previousCachePointPlacements: previousPlacements, } @@ -1178,6 +1182,15 @@ Please try: messageTemplate: `Invalid ARN format. ARN should follow the pattern: arn:aws:bedrock:region:account-id:resource-type/resource-name`, logLevel: "error", }, + TOO_MANY_IMAGES: { + patterns: ["too many images", "too many images and documents"], + messageTemplate: `Too many images in conversation. AWS Bedrock has a limit of 20 images per conversation. + +The system has automatically limited the conversation to the 20 most recent images to resolve this issue. Older images have been replaced with descriptive text placeholders. + +This is normal behavior when using the browser tool extensively, as each browser action captures a screenshot.`, + logLevel: "info", + }, VALIDATION_ERROR: { patterns: [ "input tag", @@ -1235,6 +1248,7 @@ Please check: "SERVICE_QUOTA_EXCEEDED", // Most specific - check before THROTTLING "MODEL_NOT_READY", "TOO_MANY_TOKENS", + "TOO_MANY_IMAGES", // Check for image limit errors "INTERNAL_SERVER_ERROR", "ON_DEMAND_NOT_SUPPORTED", "NOT_FOUND", diff --git a/src/api/transform/__tests__/image-limiting.spec.ts b/src/api/transform/__tests__/image-limiting.spec.ts new file mode 100644 index 0000000000..c33a702fda --- /dev/null +++ b/src/api/transform/__tests__/image-limiting.spec.ts @@ -0,0 +1,378 @@ +// npx vitest run src/api/transform/__tests__/image-limiting.spec.ts + +import { describe, it, expect } from "vitest" +import { Anthropic } from "@anthropic-ai/sdk" +import { + countImagesInMessages, + limitImagesInMessages, + exceedsBedrockImageLimit, + applyBedrockImageLimiting, + DEFAULT_BEDROCK_IMAGE_LIMIT, +} from "../image-limiting" + +describe("image-limiting", () => { + describe("countImagesInMessages", () => { + it("should count zero images in empty messages", () => { + expect(countImagesInMessages([])).toBe(0) + }) + + it("should count zero images in text-only messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello, how are you?", + }, + { + role: "assistant", + content: "I'm doing well, thank you!", + }, + ] + expect(countImagesInMessages(messages)).toBe(0) + }) + + it("should count images in mixed content messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Look at this image:" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==", + }, + }, + ], + }, + { + role: "assistant", + content: "I can see the image.", + }, + { + role: "user", + content: [ + { type: "text", text: "And another one:" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/jpeg", + data: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8A8A", + }, + }, + ], + }, + ] + expect(countImagesInMessages(messages)).toBe(2) + }) + + it("should handle string content messages", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Just text content", + }, + ] + expect(countImagesInMessages(messages)).toBe(0) + }) + }) + + describe("limitImagesInMessages", () => { + const createImageBlock = (index: number): Anthropic.ImageBlockParam => ({ + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: `image-data-${index}`, + }, + }) + + const createTextBlock = (text: string): Anthropic.TextBlockParam => ({ + type: "text", + text, + }) + + it("should not modify messages when under the limit", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [createTextBlock("Here are some images:"), createImageBlock(1), createImageBlock(2)], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 5, replaceWithText: true }) + expect(result).toEqual(messages) + }) + + it("should remove oldest images when over the limit", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [createTextBlock("First message"), createImageBlock(1), createImageBlock(2)], + }, + { + role: "assistant", + content: "I see the images.", + }, + { + role: "user", + content: [createTextBlock("Second message"), createImageBlock(3), createImageBlock(4)], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 2, replaceWithText: true }) + + // Should keep the last 2 images (3 and 4) and replace the first 2 with text + expect(countImagesInMessages(result)).toBe(2) + + // Check that first message has text replacements + const firstMessage = result[0] + expect(Array.isArray(firstMessage.content)).toBe(true) + if (Array.isArray(firstMessage.content)) { + expect(firstMessage.content[1].type).toBe("text") + expect(firstMessage.content[2].type).toBe("text") + expect((firstMessage.content[1] as Anthropic.TextBlockParam).text).toContain("Image removed") + } + + // Check that second message still has images + const thirdMessage = result[2] + expect(Array.isArray(thirdMessage.content)).toBe(true) + if (Array.isArray(thirdMessage.content)) { + expect(thirdMessage.content[1].type).toBe("image") + expect(thirdMessage.content[2].type).toBe("image") + } + }) + + it("should remove images without replacement when replaceWithText is false", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + createTextBlock("Message with images"), + createImageBlock(1), + createImageBlock(2), + createImageBlock(3), + ], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 1, replaceWithText: false }) + + expect(countImagesInMessages(result)).toBe(1) + + const message = result[0] + expect(Array.isArray(message.content)).toBe(true) + if (Array.isArray(message.content)) { + // Should have text + 1 image (2 removed completely) + expect(message.content).toHaveLength(2) + expect(message.content[0].type).toBe("text") + expect(message.content[1].type).toBe("image") + } + }) + + it("should handle edge case with exactly the limit", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [createImageBlock(1), createImageBlock(2)], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 2, replaceWithText: true }) + expect(result).toEqual(messages) + expect(countImagesInMessages(result)).toBe(2) + }) + + it("should handle messages with no images", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Just text", + }, + { + role: "assistant", + content: [createTextBlock("Also just text")], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 1, replaceWithText: true }) + expect(result).toEqual(messages) + }) + + it("should preserve message structure and other content types", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [createTextBlock("Before image"), createImageBlock(1), createTextBlock("After image")], + }, + ] + + const result = limitImagesInMessages(messages, { maxImages: 0, replaceWithText: true }) + + const message = result[0] + expect(Array.isArray(message.content)).toBe(true) + if (Array.isArray(message.content)) { + expect(message.content).toHaveLength(3) + expect(message.content[0].type).toBe("text") + expect((message.content[0] as Anthropic.TextBlockParam).text).toBe("Before image") + expect(message.content[1].type).toBe("text") + expect((message.content[1] as Anthropic.TextBlockParam).text).toContain("Image removed") + expect(message.content[2].type).toBe("text") + expect((message.content[2] as Anthropic.TextBlockParam).text).toBe("After image") + } + }) + }) + + describe("exceedsBedrockImageLimit", () => { + it("should return false for conversations under the limit", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Test" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "test-data", + }, + }, + ], + }, + ] + expect(exceedsBedrockImageLimit(messages)).toBe(false) + }) + + it("should return true for conversations over the limit", () => { + // Create 21 images (over the 20 limit) + const content: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: "Many images:" }] + for (let i = 0; i < 21; i++) { + content.push({ + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: `image-${i}`, + }, + }) + } + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content, + }, + ] + expect(exceedsBedrockImageLimit(messages)).toBe(true) + }) + }) + + describe("applyBedrockImageLimiting", () => { + it("should apply default Bedrock limiting", () => { + // Create 25 images (over the 20 limit) + const content: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: "Many images:" }] + for (let i = 0; i < 25; i++) { + content.push({ + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: `image-${i}`, + }, + }) + } + + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content, + }, + ] + + const result = applyBedrockImageLimiting(messages) + expect(countImagesInMessages(result)).toBe(DEFAULT_BEDROCK_IMAGE_LIMIT.maxImages) + }) + + it("should not modify conversations under the limit", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { type: "text", text: "Few images:" }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "image-1", + }, + }, + ], + }, + ] + + const result = applyBedrockImageLimiting(messages) + expect(result).toEqual(messages) + }) + }) + + describe("integration scenarios", () => { + it("should handle browser tool scenario with many screenshots", () => { + // Simulate a browser tool session with 25 screenshots + const messages: Anthropic.Messages.MessageParam[] = [] + + for (let i = 0; i < 25; i++) { + messages.push({ + role: "user", + content: [{ type: "text", text: `Browser action ${i + 1}` }], + }) + messages.push({ + role: "assistant", + content: [{ type: "text", text: `I'll take a screenshot` }], + }) + messages.push({ + role: "user", + content: [ + { type: "text", text: `Screenshot ${i + 1}:` }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: `screenshot-${i + 1}`, + }, + }, + ], + }) + } + + expect(countImagesInMessages(messages)).toBe(25) + + const result = applyBedrockImageLimiting(messages) + expect(countImagesInMessages(result)).toBe(20) + + // Verify that the most recent 20 images are kept + let imageCount = 0 + let foundFirstImage = false + for (let i = result.length - 1; i >= 0; i--) { + const message = result[i] + if (Array.isArray(message.content)) { + for (const block of message.content) { + if (block.type === "image") { + imageCount++ + if (!foundFirstImage) { + // The last image should be screenshot-25 + expect((block as any).source.data).toBe("screenshot-25") + foundFirstImage = true + } + } + } + } + } + expect(imageCount).toBe(20) + }) + }) +}) diff --git a/src/api/transform/image-limiting.ts b/src/api/transform/image-limiting.ts new file mode 100644 index 0000000000..23245affb6 --- /dev/null +++ b/src/api/transform/image-limiting.ts @@ -0,0 +1,128 @@ +import { Anthropic } from "@anthropic-ai/sdk" + +/** + * Configuration for image limiting + */ +export interface ImageLimitingConfig { + /** Maximum number of images to keep in conversation */ + maxImages: number + /** Whether to replace removed images with descriptive text */ + replaceWithText: boolean +} + +/** + * Default configuration for AWS Bedrock image limiting + */ +export const DEFAULT_BEDROCK_IMAGE_LIMIT: ImageLimitingConfig = { + maxImages: 20, + replaceWithText: true, +} + +/** + * Counts the total number of images across all messages in the conversation + */ +export function countImagesInMessages(messages: Anthropic.Messages.MessageParam[]): number { + let imageCount = 0 + + for (const message of messages) { + if (Array.isArray(message.content)) { + for (const block of message.content) { + if (block.type === "image") { + imageCount++ + } + } + } + } + + return imageCount +} + +/** + * Limits the number of images in a conversation by keeping only the most recent images + * and optionally replacing older images with descriptive text placeholders. + * + * This function processes messages from oldest to newest, removing images from older + * messages first when the limit is exceeded. + * + * @param messages - Array of Anthropic message parameters + * @param config - Configuration for image limiting + * @returns Modified messages with image count limited + */ +export function limitImagesInMessages( + messages: Anthropic.Messages.MessageParam[], + config: ImageLimitingConfig = DEFAULT_BEDROCK_IMAGE_LIMIT, +): Anthropic.Messages.MessageParam[] { + const totalImages = countImagesInMessages(messages) + + // If we're within the limit, return messages unchanged + if (totalImages <= config.maxImages) { + return messages + } + + // Calculate how many images we need to remove + const imagesToRemove = totalImages - config.maxImages + let imagesRemoved = 0 + + // Process messages from oldest to newest, removing images from older messages first + const modifiedMessages = messages.map((message) => { + // If we've already removed enough images, return the message unchanged + if (imagesRemoved >= imagesToRemove) { + return message + } + + // Only process messages with array content that might contain images + if (!Array.isArray(message.content)) { + return message + } + + const modifiedContent = message.content + .map((block) => { + // If we've already removed enough images, return the block unchanged + if (imagesRemoved >= imagesToRemove) { + return block + } + + // If this is an image block and we need to remove more images + if (block.type === "image") { + imagesRemoved++ + + if (config.replaceWithText) { + // Replace with descriptive text + return { + type: "text" as const, + text: "[Image removed due to conversation limit - this was a browser screenshot that has been replaced to stay within AWS Bedrock's 20-image limit]", + } + } else { + // Return null to filter out later + return null + } + } + + return block + }) + .filter((block): block is Anthropic.Messages.ContentBlockParam => block !== null) + + return { + ...message, + content: modifiedContent, + } + }) + + return modifiedMessages +} + +/** + * Checks if a conversation exceeds the image limit for AWS Bedrock + */ +export function exceedsBedrockImageLimit(messages: Anthropic.Messages.MessageParam[]): boolean { + return countImagesInMessages(messages) > DEFAULT_BEDROCK_IMAGE_LIMIT.maxImages +} + +/** + * Applies AWS Bedrock image limiting to a conversation + */ +export function applyBedrockImageLimiting( + messages: Anthropic.Messages.MessageParam[], +): Anthropic.Messages.MessageParam[] { + return limitImagesInMessages(messages, DEFAULT_BEDROCK_IMAGE_LIMIT) +}