fix: improve Bedrock error handling for throttling and streaming contexts (#4745) (#4748)

Co-authored-by: Daniel Riccio <ricciodaniel98@gmail.com>
This commit is contained in:
Hannes Rudolph 2025-06-24 09:37:19 -06:00 committed by GitHub
parent 03052f827b
commit ee751af5c6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 689 additions and 42 deletions

View file

@ -0,0 +1,551 @@
import { vi } from "vitest"
// Mock BedrockRuntimeClient and commands
const mockSend = vi.fn()
// Mock AWS SDK credential providers
vi.mock("@aws-sdk/credential-providers", () => {
return {
fromIni: vi.fn().mockReturnValue({
accessKeyId: "profile-access-key",
secretAccessKey: "profile-secret-key",
}),
}
})
vi.mock("@aws-sdk/client-bedrock-runtime", () => ({
BedrockRuntimeClient: vi.fn().mockImplementation(() => ({
send: mockSend,
})),
ConverseStreamCommand: vi.fn(),
ConverseCommand: vi.fn(),
}))
import { AwsBedrockHandler } from "../bedrock"
import { Anthropic } from "@anthropic-ai/sdk"
describe("AwsBedrockHandler Error Handling", () => {
let handler: AwsBedrockHandler
beforeEach(() => {
vi.clearAllMocks()
handler = new AwsBedrockHandler({
apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
awsAccessKey: "test-access-key",
awsSecretKey: "test-secret-key",
awsRegion: "us-east-1",
})
})
const createMockError = (options: {
message?: string
name?: string
status?: number
__type?: string
$metadata?: {
httpStatusCode?: number
requestId?: string
extendedRequestId?: string
cfId?: string
[key: string]: any // Allow additional properties
}
}): Error => {
const error = new Error(options.message || "Test error") as any
if (options.name) error.name = options.name
if (options.status) error.status = options.status
if (options.__type) error.__type = options.__type
if (options.$metadata) error.$metadata = options.$metadata
return error
}
describe("Throttling Error Detection", () => {
it("should detect throttling from HTTP 429 status code", async () => {
const throttleError = createMockError({
message: "Request failed",
status: 429,
})
mockSend.mockRejectedValueOnce(throttleError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
it("should detect throttling from AWS SDK $metadata.httpStatusCode", async () => {
const throttleError = createMockError({
message: "Request failed",
$metadata: { httpStatusCode: 429 },
})
mockSend.mockRejectedValueOnce(throttleError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
it("should detect throttling from ThrottlingException name", async () => {
const throttleError = createMockError({
message: "Request failed",
name: "ThrottlingException",
})
mockSend.mockRejectedValueOnce(throttleError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
it("should detect throttling from __type field", async () => {
const throttleError = createMockError({
message: "Request failed",
__type: "ThrottlingException",
})
mockSend.mockRejectedValueOnce(throttleError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
it("should detect throttling from 'Bedrock is unable to process your request' message", async () => {
const throttleError = createMockError({
message: "Bedrock is unable to process your request",
})
mockSend.mockRejectedValueOnce(throttleError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toMatch(/throttled or rate limited/)
}
})
it("should detect throttling from various message patterns", async () => {
const throttlingMessages = [
"Request throttled",
"Rate limit exceeded",
"Too many requests",
"Service unavailable due to high demand",
"Server is overloaded",
"System is busy",
"Please wait and try again",
]
for (const message of throttlingMessages) {
const throttleError = createMockError({ message })
mockSend.mockRejectedValueOnce(throttleError)
try {
await handler.completePrompt("test")
// Should not reach here as completePrompt should throw
throw new Error("Expected error to be thrown")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
}
})
it("should display appropriate error information for throttling errors", async () => {
const throttlingError = createMockError({
message: "Bedrock is unable to process your request",
name: "ThrottlingException",
status: 429,
$metadata: {
httpStatusCode: 429,
requestId: "12345-abcde-67890",
extendedRequestId: "extended-12345",
cfId: "cf-12345",
},
})
mockSend.mockRejectedValueOnce(throttlingError)
try {
await handler.completePrompt("test")
throw new Error("Expected error to be thrown")
} catch (error) {
// Should contain the main error message
expect(error.message).toContain("throttled or rate limited")
}
})
})
describe("Service Quota Exceeded Detection", () => {
it("should detect service quota exceeded errors", async () => {
const quotaError = createMockError({
message: "Service quota exceeded for model requests",
})
mockSend.mockRejectedValueOnce(quotaError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("Service quota exceeded")
} catch (error) {
expect(error.message).toContain("Service quota exceeded")
}
})
})
describe("Model Not Ready Detection", () => {
it("should detect model not ready errors", async () => {
const modelError = createMockError({
message: "Model is not ready, please try again later",
})
mockSend.mockRejectedValueOnce(modelError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("Model is not ready")
} catch (error) {
expect(error.message).toContain("Model is not ready")
}
})
})
describe("Internal Server Error Detection", () => {
it("should detect internal server errors", async () => {
const serverError = createMockError({
message: "Internal server error occurred",
})
mockSend.mockRejectedValueOnce(serverError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("internal server error")
} catch (error) {
expect(error.message).toContain("internal server error")
}
})
})
describe("Token Limit Detection", () => {
it("should detect enhanced token limit errors", async () => {
const tokenErrors = [
"Too many tokens in request",
"Token limit exceeded",
"Maximum context length reached",
"Context length exceeds limit",
]
for (const message of tokenErrors) {
const tokenError = createMockError({ message })
mockSend.mockRejectedValueOnce(tokenError)
try {
await handler.completePrompt("test")
// Should not reach here as completePrompt should throw
throw new Error("Expected error to be thrown")
} catch (error) {
// Either "Too many tokens" for token-specific errors or "throttled" for limit-related errors
expect(error.message).toMatch(/Too many tokens|throttled or rate limited/)
}
}
})
})
describe("Streaming Context Error Handling", () => {
it("should handle throttling errors in streaming context", async () => {
const throttleError = createMockError({
message: "Bedrock is unable to process your request",
status: 429,
})
const mockStream = {
[Symbol.asyncIterator]() {
return {
async next() {
throw throttleError
},
}
},
}
mockSend.mockResolvedValueOnce({ stream: mockStream })
const generator = handler.createMessage("system", [{ role: "user", content: "test" }])
// For throttling errors, it should throw immediately without yielding chunks
// This allows the retry mechanism to catch and handle it
await expect(async () => {
for await (const chunk of generator) {
// Should not yield any chunks for throttling errors
}
}).rejects.toThrow("Bedrock is unable to process your request")
})
it("should yield error chunks for non-throttling errors in streaming context", async () => {
const genericError = createMockError({
message: "Some other error",
status: 500,
})
const mockStream = {
[Symbol.asyncIterator]() {
return {
async next() {
throw genericError
},
}
},
}
mockSend.mockResolvedValueOnce({ stream: mockStream })
const generator = handler.createMessage("system", [{ role: "user", content: "test" }])
const chunks: any[] = []
try {
for await (const chunk of generator) {
chunks.push(chunk)
}
} catch (error) {
// Expected to throw after yielding chunks
}
// Should have yielded error chunks before throwing for non-throttling errors
expect(
chunks.some((chunk) => chunk.type === "text" && chunk.text && chunk.text.includes("Some other error")),
).toBe(true)
})
})
describe("Error Priority and Specificity", () => {
it("should prioritize HTTP status codes over message patterns", async () => {
// Error with both 429 status and generic message should be detected as throttling
const mixedError = createMockError({
message: "Some generic error message",
status: 429,
})
mockSend.mockRejectedValueOnce(mixedError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
it("should prioritize AWS error types over message patterns", async () => {
// Error with ThrottlingException name but different message should still be throttling
const specificError = createMockError({
message: "Some other error occurred",
name: "ThrottlingException",
})
mockSend.mockRejectedValueOnce(specificError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("throttled or rate limited")
} catch (error) {
expect(error.message).toContain("throttled or rate limited")
}
})
})
describe("Unknown Error Fallback", () => {
it("should still show unknown error for truly unrecognized errors", async () => {
const unknownError = createMockError({
message: "Something completely unexpected happened",
})
mockSend.mockRejectedValueOnce(unknownError)
try {
const result = await handler.completePrompt("test")
expect(result).toContain("Unknown Error")
} catch (error) {
expect(error.message).toContain("Unknown Error")
}
})
})
describe("Enhanced Error Throw for Retry System", () => {
it("should throw enhanced error messages for completePrompt to display in retry system", async () => {
const throttlingError = createMockError({
message: "Too many tokens, rate limited",
status: 429,
$metadata: {
httpStatusCode: 429,
requestId: "test-request-id-12345",
},
})
mockSend.mockRejectedValueOnce(throttlingError)
try {
await handler.completePrompt("test")
throw new Error("Expected error to be thrown")
} catch (error) {
// Should contain the verbose message template
expect(error.message).toContain("Request was throttled or rate limited")
// Should preserve original error properties
expect((error as any).status).toBe(429)
expect((error as any).$metadata.requestId).toBe("test-request-id-12345")
}
})
it("should throw enhanced error messages for createMessage streaming to display in retry system", async () => {
const tokenError = createMockError({
message: "Too many tokens in request",
name: "ValidationException",
$metadata: {
httpStatusCode: 400,
requestId: "token-error-id-67890",
extendedRequestId: "extended-12345",
},
})
const mockStream = {
[Symbol.asyncIterator]() {
return {
async next() {
throw tokenError
},
}
},
}
mockSend.mockResolvedValueOnce({ stream: mockStream })
try {
const stream = handler.createMessage("system", [{ role: "user", content: "test" }])
for await (const chunk of stream) {
// Should not reach here as it should throw an error
}
throw new Error("Expected error to be thrown")
} catch (error) {
// Should contain error codes (note: this will be caught by the non-throttling error path)
expect(error.message).toContain("Too many tokens")
// Should preserve original error properties
expect(error.name).toBe("ValidationException")
expect((error as any).$metadata.requestId).toBe("token-error-id-67890")
}
})
})
describe("Edge Case Test Coverage", () => {
it("should handle concurrent throttling errors correctly", async () => {
const throttlingError = createMockError({
message: "Bedrock is unable to process your request",
status: 429,
})
// Setup multiple concurrent requests that will all fail with throttling
mockSend.mockRejectedValue(throttlingError)
// Execute multiple concurrent requests
const promises = Array.from({ length: 5 }, () => handler.completePrompt("test"))
// All should throw with throttling error
const results = await Promise.allSettled(promises)
results.forEach((result) => {
expect(result.status).toBe("rejected")
if (result.status === "rejected") {
expect(result.reason.message).toContain("throttled or rate limited")
}
})
})
it("should handle mixed error scenarios with both throttling and other indicators", async () => {
// Error with both 429 status (throttling) and validation error message
const mixedError = createMockError({
message: "ValidationException: Your input is invalid, but also rate limited",
name: "ValidationException",
status: 429,
$metadata: {
httpStatusCode: 429,
requestId: "mixed-error-id",
},
})
mockSend.mockRejectedValueOnce(mixedError)
try {
await handler.completePrompt("test")
} catch (error) {
// Should be treated as throttling due to 429 status taking priority
expect(error.message).toContain("throttled or rate limited")
// Should still preserve metadata
expect((error as any).$metadata?.requestId).toBe("mixed-error-id")
}
})
it("should handle rapid successive retries in streaming context", async () => {
const throttlingError = createMockError({
message: "ThrottlingException",
name: "ThrottlingException",
})
// Mock stream that throws immediately
const mockStream = {
// eslint-disable-next-line require-yield
[Symbol.asyncIterator]: async function* () {
throw throttlingError
},
}
mockSend.mockResolvedValueOnce({ stream: mockStream })
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "test" }]
try {
// Should throw immediately without yielding any chunks
const stream = handler.createMessage("", messages)
const chunks = []
for await (const chunk of stream) {
chunks.push(chunk)
}
// Should not reach here
expect(chunks).toHaveLength(0)
} catch (error) {
// Error should be thrown immediately for retry mechanism
// The error might be a TypeError if the stream iterator fails
expect(error).toBeDefined()
// The important thing is that it throws immediately without yielding chunks
}
})
it("should validate error properties exist before accessing them", async () => {
// Error with unusual structure
const unusualError = {
message: "Error with unusual structure",
// Missing typical properties like name, status, etc.
}
mockSend.mockRejectedValueOnce(unusualError)
try {
await handler.completePrompt("test")
} catch (error) {
// Should handle gracefully without accessing undefined properties
expect(error.message).toContain("Unknown Error")
// Should not have undefined values in the error message
expect(error.message).not.toContain("undefined")
}
})
})
})

View file

@ -561,16 +561,47 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
// Clear timeout on error
clearTimeout(timeoutId)
// Use the extracted error handling method for all errors
// Check if this is a throttling error that should trigger retry logic
const errorType = this.getErrorType(error)
// For throttling errors, throw immediately without yielding chunks
// This allows the retry mechanism in attemptApiRequest() to catch and handle it
// The retry logic in Task.ts (around line 1817) expects errors to be thrown
// on the first chunk for proper exponential backoff behavior
if (errorType === "THROTTLING") {
if (error instanceof Error) {
throw error
} else {
throw new Error("Throttling error occurred")
}
}
// For non-throttling errors, use the standard error handling with chunks
const errorChunks = this.handleBedrockError(error, true) // true for streaming context
// Yield each chunk individually to ensure type compatibility
for (const chunk of errorChunks) {
yield chunk as any // Cast to any to bypass type checking since we know the structure is correct
}
// Re-throw the error
// Re-throw with enhanced error message for retry system
const enhancedErrorMessage = this.formatErrorMessage(error, this.getErrorType(error), true)
if (error instanceof Error) {
throw error
const enhancedError = new Error(enhancedErrorMessage)
// Preserve important properties from the original error
enhancedError.name = error.name
// Validate and preserve status property
if ("status" in error && typeof (error as any).status === "number") {
;(enhancedError as any).status = (error as any).status
}
// Validate and preserve $metadata property
if (
"$metadata" in error &&
typeof (error as any).$metadata === "object" &&
(error as any).$metadata !== null
) {
;(enhancedError as any).$metadata = (error as any).$metadata
}
throw enhancedError
} else {
throw new Error("An unknown error occurred")
}
@ -638,7 +669,26 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
const errorResult = this.handleBedrockError(error, false) // false for non-streaming context
// Since we're in a non-streaming context, we know the result is a string
const errorMessage = errorResult as string
throw new Error(errorMessage)
// Create enhanced error for retry system
const enhancedError = new Error(errorMessage)
if (error instanceof Error) {
// Preserve important properties from the original error
enhancedError.name = error.name
// Validate and preserve status property
if ("status" in error && typeof (error as any).status === "number") {
;(enhancedError as any).status = (error as any).status
}
// Validate and preserve $metadata property
if (
"$metadata" in error &&
typeof (error as any).$metadata === "object" &&
(error as any).$metadata !== null
) {
;(enhancedError as any).$metadata = (error as any).$metadata
}
}
throw enhancedError
}
}
@ -1035,19 +1085,32 @@ Please verify:
logLevel: "error",
},
THROTTLING: {
patterns: ["throttl", "rate", "limit"],
patterns: [
"throttl",
"rate",
"limit",
"bedrock is unable to process your request", // AWS Bedrock specific throttling message
"please wait",
"quota exceeded",
"service unavailable",
"busy",
"overloaded",
"too many requests",
"request limit",
"concurrent requests",
],
messageTemplate: `Request was throttled or rate limited. Please try:
1. Reducing the frequency of requests
2. If using a provisioned model, check its throughput settings
3. Contact AWS support to request a quota increase if needed
{formattedErrorDetails}
`,
logLevel: "error",
},
TOO_MANY_TOKENS: {
patterns: ["too many tokens"],
patterns: ["too many tokens", "token limit exceeded", "context length", "maximum context length"],
messageTemplate: `"Too many tokens" error detected.
Possible Causes:
1. Input exceeds model's context window limit
@ -1060,7 +1123,49 @@ Suggestions:
2. Split your request into smaller chunks
3. Use a model with a larger context window
4. If rate limited, reduce request frequency
5. Check your Amazon Bedrock quotas and limits`,
5. Check your Amazon Bedrock quotas and limits
`,
logLevel: "error",
},
SERVICE_QUOTA_EXCEEDED: {
patterns: ["service quota exceeded", "service quota", "quota exceeded for model"],
messageTemplate: `Service quota exceeded. This error indicates you've reached AWS service limits.
Please try:
1. Contact AWS support to request a quota increase
2. Reduce request frequency temporarily
3. Check your AWS Bedrock quotas in the AWS console
4. Consider using a different model or region with available capacity
`,
logLevel: "error",
},
MODEL_NOT_READY: {
patterns: ["model not ready", "model is not ready", "provisioned throughput not ready", "model loading"],
messageTemplate: `Model is not ready or still loading. This can happen with:
1. Provisioned throughput models that are still initializing
2. Custom models that are being loaded
3. Models that are temporarily unavailable
Please try:
1. Wait a few minutes and retry
2. Check the model status in AWS Bedrock console
3. Verify the model is properly provisioned
`,
logLevel: "error",
},
INTERNAL_SERVER_ERROR: {
patterns: ["internal server error", "internal error", "server error", "service error"],
messageTemplate: `AWS Bedrock internal server error. This is a temporary service issue.
Please try:
1. Retry the request after a brief delay
2. If the error persists, check AWS service health
3. Contact AWS support if the issue continues
`,
logLevel: "error",
},
ON_DEMAND_NOT_SUPPORTED: {
@ -1119,12 +1224,34 @@ Please check:
return "GENERIC"
}
// Check for HTTP 429 status code (Too Many Requests)
if ((error as any).status === 429 || (error as any).$metadata?.httpStatusCode === 429) {
return "THROTTLING"
}
// Check for AWS Bedrock specific throttling exception names
if ((error as any).name === "ThrottlingException" || (error as any).__type === "ThrottlingException") {
return "THROTTLING"
}
const errorMessage = error.message.toLowerCase()
const errorName = error.name.toLowerCase()
// Check each error type's patterns
for (const [errorType, definition] of Object.entries(AwsBedrockHandler.ERROR_TYPES)) {
if (errorType === "GENERIC") continue // Skip the generic type
// Check each error type's patterns in order of specificity (most specific first)
const errorTypeOrder = [
"SERVICE_QUOTA_EXCEEDED", // Most specific - check before THROTTLING
"MODEL_NOT_READY",
"TOO_MANY_TOKENS",
"INTERNAL_SERVER_ERROR",
"ON_DEMAND_NOT_SUPPORTED",
"NOT_FOUND",
"ACCESS_DENIED",
"THROTTLING", // Less specific - check after more specific patterns
]
for (const errorType of errorTypeOrder) {
const definition = AwsBedrockHandler.ERROR_TYPES[errorType]
if (!definition) continue
// If any pattern matches in either message or name, return this error type
if (definition.patterns.some((pattern) => errorMessage.includes(pattern) || errorName.includes(pattern))) {
@ -1153,37 +1280,6 @@ Please check:
const modelConfig = this.getModel()
templateVars.modelId = modelConfig.id
templateVars.contextWindow = String(modelConfig.info.contextWindow || "unknown")
// Format error details
const errorDetails: Record<string, any> = {}
Object.getOwnPropertyNames(error).forEach((prop) => {
if (prop !== "stack") {
errorDetails[prop] = (error as any)[prop]
}
})
// Safely stringify error details to avoid circular references
templateVars.formattedErrorDetails = Object.entries(errorDetails)
.map(([key, value]) => {
let valueStr
if (typeof value === "object" && value !== null) {
try {
// Use a replacer function to handle circular references
valueStr = JSON.stringify(value, (k, v) => {
if (k && typeof v === "object" && v !== null) {
return "[Object]"
}
return v
})
} catch (e) {
valueStr = "[Complex Object]"
}
} else {
valueStr = String(value)
}
return `- ${key}: ${valueStr}`
})
.join("\n")
}
// Add context-specific template variables