diff --git a/.changeset/.mighty-tools-remain.md b/.changeset/.mighty-tools-remain.md new file mode 100644 index 0000000000..b66bf2073e --- /dev/null +++ b/.changeset/.mighty-tools-remain.md @@ -0,0 +1,7 @@ +--- +"claude-dev": minor +--- +Add Bedrock prompt caching support (optional). + +This feature protected under checkbox because it is not yet rolled out to everyone, and if you will try to send cache headers, and its not enabled for you, you will get error. + diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index 59a85fd178..7a17fd6260 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -19,9 +19,15 @@ export class AwsBedrockHandler implements ApiHandler { // cross region inference requires prefixing the model id with the region let modelId = await this.getModelId() - // create anthropic client, using sessions created or renewed after this handler's + // Get model info and message indices for caching + const model = this.getModel() + const userMsgIndices = messages.reduce((acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), [] as number[]) + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + + // Create anthropic client, using sessions created or renewed after this handler's // initialization, and allowing for session renewal if necessary as well - let client = await this.getClient() + const client = await this.getClient() const stream = await client.messages.create({ model: modelId, @@ -32,6 +38,7 @@ export class AwsBedrockHandler implements ApiHandler { messages, stream: true, }) + for await (const chunk of stream) { switch (chunk.type) { case "message_start": @@ -40,6 +47,8 @@ export class AwsBedrockHandler implements ApiHandler { type: "usage", inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0, + cacheWriteTokens: usage.cache_creation_input_tokens || undefined, + cacheReadTokens: usage.cache_read_input_tokens || undefined, } break case "message_delta": @@ -49,7 +58,6 @@ export class AwsBedrockHandler implements ApiHandler { outputTokens: chunk.usage.output_tokens || 0, } break - case "content_block_start": switch (chunk.content_block.type) { case "text": @@ -129,11 +137,9 @@ export class AwsBedrockHandler implements ApiHandler { return `us.${this.getModel().id}` case "eu-": return `eu.${this.getModel().id}` - break default: // cross region inference is not supported in this region, falling back to default model return this.getModel().id - break } } return this.getModel().id @@ -150,7 +156,7 @@ export class AwsBedrockHandler implements ApiHandler { } } - private static async setEnv(key: string, value: string | undefined) { + private static setEnv(key: string, value: string | undefined) { if (key !== "" && value !== undefined) { process.env[key] = value } diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index d3f8322ee4..c424ecbd2e 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -65,6 +65,7 @@ type GlobalStateKey = | "apiModelId" | "awsRegion" | "awsUseCrossRegionInference" + | "awsBedrockUsePromptCache" | "awsProfile" | "awsUseProfile" | "vertexProjectId" @@ -565,6 +566,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { awsSessionToken, awsRegion, awsUseCrossRegionInference, + awsBedrockUsePromptCache, awsProfile, awsUseProfile, vertexProjectId, @@ -607,6 +609,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { await this.storeSecret("awsSessionToken", awsSessionToken) await this.updateGlobalState("awsRegion", awsRegion) await this.updateGlobalState("awsUseCrossRegionInference", awsUseCrossRegionInference) + await this.updateGlobalState("awsBedrockUsePromptCache", awsBedrockUsePromptCache) await this.updateGlobalState("awsProfile", awsProfile) await this.updateGlobalState("awsUseProfile", awsUseProfile) await this.updateGlobalState("vertexProjectId", vertexProjectId) @@ -1848,6 +1851,7 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont awsSessionToken, awsRegion, awsUseCrossRegionInference, + awsBedrockUsePromptCache, awsProfile, awsUseProfile, vertexProjectId, @@ -1903,6 +1907,7 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont this.getSecret("awsSessionToken") as Promise, this.getGlobalState("awsRegion") as Promise, this.getGlobalState("awsUseCrossRegionInference") as Promise, + this.getGlobalState("awsBedrockUsePromptCache") as Promise, this.getGlobalState("awsProfile") as Promise, this.getGlobalState("awsUseProfile") as Promise, this.getGlobalState("vertexProjectId") as Promise, @@ -1981,6 +1986,7 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont awsSessionToken, awsRegion, awsUseCrossRegionInference, + awsBedrockUsePromptCache, awsProfile, awsUseProfile, vertexProjectId, diff --git a/src/shared/api.ts b/src/shared/api.ts index 7a052a8d5e..a57c5887d2 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -32,6 +32,7 @@ export interface ApiHandlerOptions { awsSessionToken?: string awsRegion?: string awsUseCrossRegionInference?: boolean + awsBedrockUsePromptCache?: boolean awsUseProfile?: boolean awsProfile?: string vertexProjectId?: string @@ -160,17 +161,21 @@ export const bedrockModels = { contextWindow: 200_000, supportsImages: true, supportsComputerUse: true, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 3.0, outputPrice: 15.0, + cacheWritesPrice: 3.75, + cacheReadsPrice: 0.3, }, "anthropic.claude-3-5-haiku-20241022-v1:0": { maxTokens: 8192, contextWindow: 200_000, supportsImages: false, - supportsPromptCache: false, + supportsPromptCache: true, inputPrice: 1.0, outputPrice: 5.0, + cacheWritesPrice: 1.0, + cacheReadsPrice: 0.08, }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { maxTokens: 8192, diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index 67724a816b..baf39369f1 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -537,17 +537,35 @@ const ApiOptions = ({ showModelOptions, apiErrorMessage, modelIdErrorMessage, is {/* us-gov-east-1 */} - { - const isChecked = e.target.checked === true - setApiConfiguration({ - ...apiConfiguration, - awsUseCrossRegionInference: isChecked, - }) - }}> - Use cross-region inference - +
+ { + const isChecked = e.target.checked === true + setApiConfiguration({ + ...apiConfiguration, + awsUseCrossRegionInference: isChecked, + }) + }}> + Use cross-region inference + + + {selectedModelInfo.supportsPromptCache && ( + <> + { + const isChecked = e.target.checked === true + setApiConfiguration({ + ...apiConfiguration, + awsBedrockUsePromptCache: isChecked, + }) + }}> + Use prompt caching (Beta) + + + )} +