mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-08-28 05:27:24 +00:00
feat: limit images to 20 for AWS Bedrock browser tool usage
- Add TOO_MANY_IMAGES error type to Bedrock error handling - Create image limiting utility that keeps most recent 20 images - Replace older images with descriptive text placeholders - Integrate image limiting into Bedrock message conversion - Add comprehensive tests for image limiting functionality Fixes #6348
This commit is contained in:
Roo Code
parent
75f93c41cf
commit
ca26779b2b
3 changed files with 522 additions and 2 deletions
|
|
@ -30,6 +30,7 @@ import { MultiPointStrategy } from "../transform/cache-strategy/multi-point-stra
|
|||
import { ModelInfo as CacheModelInfo } from "../transform/cache-strategy/types"
|
||||
import { convertToBedrockConverseMessages as sharedConverter } from "../transform/bedrock-converse-format"
|
||||
import { getModelParams } from "../transform/model-params"
|
||||
import { applyBedrockImageLimiting } from "../transform/image-limiting"
|
||||
import { shouldUseReasoningBudget } from "../../shared/api"
|
||||
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
|
||||
|
|
@ -706,8 +707,11 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
|
|||
modelInfo?: any,
|
||||
conversationId?: string, // Optional conversation ID to track cache points across messages
|
||||
): { system: SystemContentBlock[]; messages: Message[] } {
|
||||
// Apply image limiting before conversion to prevent "too many images" errors
|
||||
const limitedMessages = applyBedrockImageLimiting(anthropicMessages as Anthropic.Messages.MessageParam[])
|
||||
|
||||
// First convert messages using shared converter for proper image handling
|
||||
const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[])
|
||||
const convertedMessages = sharedConverter(limitedMessages)
|
||||
|
||||
// If prompt caching is disabled, return the converted messages directly
|
||||
if (!usePromptCache) {
|
||||
|
|
@ -737,7 +741,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
|
|||
const config = {
|
||||
modelInfo: cacheModelInfo,
|
||||
systemPrompt: systemMessage,
|
||||
messages: anthropicMessages as Anthropic.Messages.MessageParam[],
|
||||
messages: limitedMessages,
|
||||
usePromptCache,
|
||||
previousCachePointPlacements: previousPlacements,
|
||||
}
|
||||
|
|
@ -1178,6 +1182,15 @@ Please try:
|
|||
messageTemplate: `Invalid ARN format. ARN should follow the pattern: arn:aws:bedrock:region:account-id:resource-type/resource-name`,
|
||||
logLevel: "error",
|
||||
},
|
||||
TOO_MANY_IMAGES: {
|
||||
patterns: ["too many images", "too many images and documents"],
|
||||
messageTemplate: `Too many images in conversation. AWS Bedrock has a limit of 20 images per conversation.
|
||||
|
||||
The system has automatically limited the conversation to the 20 most recent images to resolve this issue. Older images have been replaced with descriptive text placeholders.
|
||||
|
||||
This is normal behavior when using the browser tool extensively, as each browser action captures a screenshot.`,
|
||||
logLevel: "info",
|
||||
},
|
||||
VALIDATION_ERROR: {
|
||||
patterns: [
|
||||
"input tag",
|
||||
|
|
@ -1235,6 +1248,7 @@ Please check:
|
|||
"SERVICE_QUOTA_EXCEEDED", // Most specific - check before THROTTLING
|
||||
"MODEL_NOT_READY",
|
||||
"TOO_MANY_TOKENS",
|
||||
"TOO_MANY_IMAGES", // Check for image limit errors
|
||||
"INTERNAL_SERVER_ERROR",
|
||||
"ON_DEMAND_NOT_SUPPORTED",
|
||||
"NOT_FOUND",
|
||||
|
|
|
|||
378
src/api/transform/__tests__/image-limiting.spec.ts
Normal file
378
src/api/transform/__tests__/image-limiting.spec.ts
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
// npx vitest run src/api/transform/__tests__/image-limiting.spec.ts
|
||||
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
import {
|
||||
countImagesInMessages,
|
||||
limitImagesInMessages,
|
||||
exceedsBedrockImageLimit,
|
||||
applyBedrockImageLimiting,
|
||||
DEFAULT_BEDROCK_IMAGE_LIMIT,
|
||||
} from "../image-limiting"
|
||||
|
||||
describe("image-limiting", () => {
|
||||
describe("countImagesInMessages", () => {
|
||||
it("should count zero images in empty messages", () => {
|
||||
expect(countImagesInMessages([])).toBe(0)
|
||||
})
|
||||
|
||||
it("should count zero images in text-only messages", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello, how are you?",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "I'm doing well, thank you!",
|
||||
},
|
||||
]
|
||||
expect(countImagesInMessages(messages)).toBe(0)
|
||||
})
|
||||
|
||||
it("should count images in mixed content messages", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Look at this image:" },
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "I can see the image.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "And another one:" },
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/jpeg",
|
||||
data: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8A8A",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
expect(countImagesInMessages(messages)).toBe(2)
|
||||
})
|
||||
|
||||
it("should handle string content messages", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: "Just text content",
|
||||
},
|
||||
]
|
||||
expect(countImagesInMessages(messages)).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe("limitImagesInMessages", () => {
|
||||
const createImageBlock = (index: number): Anthropic.ImageBlockParam => ({
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: `image-data-${index}`,
|
||||
},
|
||||
})
|
||||
|
||||
const createTextBlock = (text: string): Anthropic.TextBlockParam => ({
|
||||
type: "text",
|
||||
text,
|
||||
})
|
||||
|
||||
it("should not modify messages when under the limit", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [createTextBlock("Here are some images:"), createImageBlock(1), createImageBlock(2)],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 5, replaceWithText: true })
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it("should remove oldest images when over the limit", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [createTextBlock("First message"), createImageBlock(1), createImageBlock(2)],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "I see the images.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [createTextBlock("Second message"), createImageBlock(3), createImageBlock(4)],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 2, replaceWithText: true })
|
||||
|
||||
// Should keep the last 2 images (3 and 4) and replace the first 2 with text
|
||||
expect(countImagesInMessages(result)).toBe(2)
|
||||
|
||||
// Check that first message has text replacements
|
||||
const firstMessage = result[0]
|
||||
expect(Array.isArray(firstMessage.content)).toBe(true)
|
||||
if (Array.isArray(firstMessage.content)) {
|
||||
expect(firstMessage.content[1].type).toBe("text")
|
||||
expect(firstMessage.content[2].type).toBe("text")
|
||||
expect((firstMessage.content[1] as Anthropic.TextBlockParam).text).toContain("Image removed")
|
||||
}
|
||||
|
||||
// Check that second message still has images
|
||||
const thirdMessage = result[2]
|
||||
expect(Array.isArray(thirdMessage.content)).toBe(true)
|
||||
if (Array.isArray(thirdMessage.content)) {
|
||||
expect(thirdMessage.content[1].type).toBe("image")
|
||||
expect(thirdMessage.content[2].type).toBe("image")
|
||||
}
|
||||
})
|
||||
|
||||
it("should remove images without replacement when replaceWithText is false", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
createTextBlock("Message with images"),
|
||||
createImageBlock(1),
|
||||
createImageBlock(2),
|
||||
createImageBlock(3),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 1, replaceWithText: false })
|
||||
|
||||
expect(countImagesInMessages(result)).toBe(1)
|
||||
|
||||
const message = result[0]
|
||||
expect(Array.isArray(message.content)).toBe(true)
|
||||
if (Array.isArray(message.content)) {
|
||||
// Should have text + 1 image (2 removed completely)
|
||||
expect(message.content).toHaveLength(2)
|
||||
expect(message.content[0].type).toBe("text")
|
||||
expect(message.content[1].type).toBe("image")
|
||||
}
|
||||
})
|
||||
|
||||
it("should handle edge case with exactly the limit", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [createImageBlock(1), createImageBlock(2)],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 2, replaceWithText: true })
|
||||
expect(result).toEqual(messages)
|
||||
expect(countImagesInMessages(result)).toBe(2)
|
||||
})
|
||||
|
||||
it("should handle messages with no images", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: "Just text",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: [createTextBlock("Also just text")],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 1, replaceWithText: true })
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it("should preserve message structure and other content types", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [createTextBlock("Before image"), createImageBlock(1), createTextBlock("After image")],
|
||||
},
|
||||
]
|
||||
|
||||
const result = limitImagesInMessages(messages, { maxImages: 0, replaceWithText: true })
|
||||
|
||||
const message = result[0]
|
||||
expect(Array.isArray(message.content)).toBe(true)
|
||||
if (Array.isArray(message.content)) {
|
||||
expect(message.content).toHaveLength(3)
|
||||
expect(message.content[0].type).toBe("text")
|
||||
expect((message.content[0] as Anthropic.TextBlockParam).text).toBe("Before image")
|
||||
expect(message.content[1].type).toBe("text")
|
||||
expect((message.content[1] as Anthropic.TextBlockParam).text).toContain("Image removed")
|
||||
expect(message.content[2].type).toBe("text")
|
||||
expect((message.content[2] as Anthropic.TextBlockParam).text).toBe("After image")
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe("exceedsBedrockImageLimit", () => {
|
||||
it("should return false for conversations under the limit", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Test" },
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: "test-data",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
expect(exceedsBedrockImageLimit(messages)).toBe(false)
|
||||
})
|
||||
|
||||
it("should return true for conversations over the limit", () => {
|
||||
// Create 21 images (over the 20 limit)
|
||||
const content: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: "Many images:" }]
|
||||
for (let i = 0; i < 21; i++) {
|
||||
content.push({
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: `image-${i}`,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content,
|
||||
},
|
||||
]
|
||||
expect(exceedsBedrockImageLimit(messages)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe("applyBedrockImageLimiting", () => {
|
||||
it("should apply default Bedrock limiting", () => {
|
||||
// Create 25 images (over the 20 limit)
|
||||
const content: Anthropic.Messages.ContentBlockParam[] = [{ type: "text", text: "Many images:" }]
|
||||
for (let i = 0; i < 25; i++) {
|
||||
content.push({
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: `image-${i}`,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content,
|
||||
},
|
||||
]
|
||||
|
||||
const result = applyBedrockImageLimiting(messages)
|
||||
expect(countImagesInMessages(result)).toBe(DEFAULT_BEDROCK_IMAGE_LIMIT.maxImages)
|
||||
})
|
||||
|
||||
it("should not modify conversations under the limit", () => {
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Few images:" },
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: "image-1",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
const result = applyBedrockImageLimiting(messages)
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
})
|
||||
|
||||
describe("integration scenarios", () => {
|
||||
it("should handle browser tool scenario with many screenshots", () => {
|
||||
// Simulate a browser tool session with 25 screenshots
|
||||
const messages: Anthropic.Messages.MessageParam[] = []
|
||||
|
||||
for (let i = 0; i < 25; i++) {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: `Browser action ${i + 1}` }],
|
||||
})
|
||||
messages.push({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: `I'll take a screenshot` }],
|
||||
})
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: `Screenshot ${i + 1}:` },
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: `screenshot-${i + 1}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
expect(countImagesInMessages(messages)).toBe(25)
|
||||
|
||||
const result = applyBedrockImageLimiting(messages)
|
||||
expect(countImagesInMessages(result)).toBe(20)
|
||||
|
||||
// Verify that the most recent 20 images are kept
|
||||
let imageCount = 0
|
||||
let foundFirstImage = false
|
||||
for (let i = result.length - 1; i >= 0; i--) {
|
||||
const message = result[i]
|
||||
if (Array.isArray(message.content)) {
|
||||
for (const block of message.content) {
|
||||
if (block.type === "image") {
|
||||
imageCount++
|
||||
if (!foundFirstImage) {
|
||||
// The last image should be screenshot-25
|
||||
expect((block as any).source.data).toBe("screenshot-25")
|
||||
foundFirstImage = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(imageCount).toBe(20)
|
||||
})
|
||||
})
|
||||
})
|
||||
128
src/api/transform/image-limiting.ts
Normal file
128
src/api/transform/image-limiting.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
|
||||
/**
|
||||
* Configuration for image limiting
|
||||
*/
|
||||
export interface ImageLimitingConfig {
|
||||
/** Maximum number of images to keep in conversation */
|
||||
maxImages: number
|
||||
/** Whether to replace removed images with descriptive text */
|
||||
replaceWithText: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration for AWS Bedrock image limiting
|
||||
*/
|
||||
export const DEFAULT_BEDROCK_IMAGE_LIMIT: ImageLimitingConfig = {
|
||||
maxImages: 20,
|
||||
replaceWithText: true,
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts the total number of images across all messages in the conversation
|
||||
*/
|
||||
export function countImagesInMessages(messages: Anthropic.Messages.MessageParam[]): number {
|
||||
let imageCount = 0
|
||||
|
||||
for (const message of messages) {
|
||||
if (Array.isArray(message.content)) {
|
||||
for (const block of message.content) {
|
||||
if (block.type === "image") {
|
||||
imageCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return imageCount
|
||||
}
|
||||
|
||||
/**
|
||||
* Limits the number of images in a conversation by keeping only the most recent images
|
||||
* and optionally replacing older images with descriptive text placeholders.
|
||||
*
|
||||
* This function processes messages from oldest to newest, removing images from older
|
||||
* messages first when the limit is exceeded.
|
||||
*
|
||||
* @param messages - Array of Anthropic message parameters
|
||||
* @param config - Configuration for image limiting
|
||||
* @returns Modified messages with image count limited
|
||||
*/
|
||||
export function limitImagesInMessages(
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
config: ImageLimitingConfig = DEFAULT_BEDROCK_IMAGE_LIMIT,
|
||||
): Anthropic.Messages.MessageParam[] {
|
||||
const totalImages = countImagesInMessages(messages)
|
||||
|
||||
// If we're within the limit, return messages unchanged
|
||||
if (totalImages <= config.maxImages) {
|
||||
return messages
|
||||
}
|
||||
|
||||
// Calculate how many images we need to remove
|
||||
const imagesToRemove = totalImages - config.maxImages
|
||||
let imagesRemoved = 0
|
||||
|
||||
// Process messages from oldest to newest, removing images from older messages first
|
||||
const modifiedMessages = messages.map((message) => {
|
||||
// If we've already removed enough images, return the message unchanged
|
||||
if (imagesRemoved >= imagesToRemove) {
|
||||
return message
|
||||
}
|
||||
|
||||
// Only process messages with array content that might contain images
|
||||
if (!Array.isArray(message.content)) {
|
||||
return message
|
||||
}
|
||||
|
||||
const modifiedContent = message.content
|
||||
.map((block) => {
|
||||
// If we've already removed enough images, return the block unchanged
|
||||
if (imagesRemoved >= imagesToRemove) {
|
||||
return block
|
||||
}
|
||||
|
||||
// If this is an image block and we need to remove more images
|
||||
if (block.type === "image") {
|
||||
imagesRemoved++
|
||||
|
||||
if (config.replaceWithText) {
|
||||
// Replace with descriptive text
|
||||
return {
|
||||
type: "text" as const,
|
||||
text: "[Image removed due to conversation limit - this was a browser screenshot that has been replaced to stay within AWS Bedrock's 20-image limit]",
|
||||
}
|
||||
} else {
|
||||
// Return null to filter out later
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
return block
|
||||
})
|
||||
.filter((block): block is Anthropic.Messages.ContentBlockParam => block !== null)
|
||||
|
||||
return {
|
||||
...message,
|
||||
content: modifiedContent,
|
||||
}
|
||||
})
|
||||
|
||||
return modifiedMessages
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a conversation exceeds the image limit for AWS Bedrock
|
||||
*/
|
||||
export function exceedsBedrockImageLimit(messages: Anthropic.Messages.MessageParam[]): boolean {
|
||||
return countImagesInMessages(messages) > DEFAULT_BEDROCK_IMAGE_LIMIT.maxImages
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies AWS Bedrock image limiting to a conversation
|
||||
*/
|
||||
export function applyBedrockImageLimiting(
|
||||
messages: Anthropic.Messages.MessageParam[],
|
||||
): Anthropic.Messages.MessageParam[] {
|
||||
return limitImagesInMessages(messages, DEFAULT_BEDROCK_IMAGE_LIMIT)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue