From 4a3ebb52ddd02ba6c27f49cd3fcdc24da126f436 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Wed, 11 Feb 2026 13:11:22 -0700 Subject: [PATCH] refactor: unify prompt caching via AI SDK with legacy migration --- packages/types/src/provider-settings.ts | 5 +- src/api/providers/__tests__/anthropic.spec.ts | 21 ++ src/api/providers/__tests__/bedrock.spec.ts | 70 ++++++ src/api/providers/__tests__/minimax.spec.ts | 13 ++ .../providers/__tests__/openai-native.spec.ts | 57 +++++ src/api/providers/anthropic-vertex.ts | 71 ++---- src/api/providers/anthropic.ts | 60 ++--- src/api/providers/bedrock.ts | 116 ++-------- src/api/providers/minimax.ts | 51 ++--- src/api/providers/openai-native.ts | 33 ++- .../transform/__tests__/prompt-cache.spec.ts | 210 ++++++++++++++++++ src/api/transform/prompt-cache.ts | 175 +++++++++++++++ src/core/config/ContextProxy.ts | 50 +++++ src/core/config/ProviderSettingsManager.ts | 26 ++- .../config/__tests__/ContextProxy.spec.ts | 33 ++- .../__tests__/ProviderSettingsManager.spec.ts | 39 ++++ .../migrateLegacyPromptCacheSettings.spec.ts | 67 ++++++ .../migrateLegacyPromptCacheSettings.ts | 50 +++++ .../src/components/settings/ApiOptions.tsx | 36 ++- .../settings/__tests__/ApiOptions.spec.tsx | 37 +++ .../components/settings/providers/Bedrock.tsx | 28 +-- .../components/settings/providers/LiteLLM.tsx | 25 +-- .../providers/__tests__/Bedrock.spec.tsx | 16 ++ .../src/context/ExtensionStateContext.tsx | 3 - 24 files changed, 969 insertions(+), 323 deletions(-) create mode 100644 src/api/transform/__tests__/prompt-cache.spec.ts create mode 100644 src/api/transform/prompt-cache.ts create mode 100644 src/core/config/__tests__/migrateLegacyPromptCacheSettings.spec.ts create mode 100644 src/core/config/migrateLegacyPromptCacheSettings.ts diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index bf3364d38d..a4190182cf 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -180,6 +180,9 @@ const baseProviderSettingsSchema = z.object({ modelTemperature: z.number().nullish(), rateLimitSeconds: z.number().optional(), consecutiveMistakeLimit: z.number().min(0).optional(), + promptCachingEnabled: z.boolean().optional(), + promptCachingStrategy: z.enum(["conservative", "balanced", "aggressive"]).optional(), + promptCachingProviderOverrides: z.record(z.string(), z.boolean()).optional(), // Model reasoning. enableReasoningEffort: z.boolean().optional(), @@ -217,7 +220,6 @@ const bedrockSchema = apiModelIdProviderModelSchema.extend({ awsRegion: z.string().optional(), awsUseCrossRegionInference: z.boolean().optional(), awsUseGlobalInference: z.boolean().optional(), // Enable Global Inference profile routing when supported - awsUsePromptCache: z.boolean().optional(), awsProfile: z.string().optional(), awsUseProfile: z.boolean().optional(), awsApiKey: z.string().optional(), @@ -340,7 +342,6 @@ const litellmSchema = baseProviderSettingsSchema.extend({ litellmBaseUrl: z.string().optional(), litellmApiKey: z.string().optional(), litellmModelId: z.string().optional(), - litellmUsePromptCache: z.boolean().optional(), }) const sambaNovaSchema = apiModelIdProviderModelSchema.extend({ diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 75fcb119ad..47dd5f4e64 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -420,6 +420,27 @@ describe("AnthropicHandler", () => { const systemMessages = callArgs.messages.filter((m: any) => m.role === "system") expect(systemMessages).toHaveLength(0) }) + + it("should disable prompt caching when globally disabled", async () => { + setupStreamTextMock([{ type: "text-delta", text: "test" }]) + + const cacheDisabledHandler = new AnthropicHandler({ + ...mockOptions, + promptCachingEnabled: false, + }) + + const stream = cacheDisabledHandler.createMessage(systemPrompt, [ + { role: "user", content: [{ type: "text" as const, text: "hello" }] }, + ]) + + for await (const _chunk of stream) { + // Consume + } + + const callArgs = mockStreamText.mock.calls[0]![0] + expect(callArgs.systemProviderOptions).toBeUndefined() + expect(callArgs.messages[0].providerOptions).toBeUndefined() + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/__tests__/bedrock.spec.ts b/src/api/providers/__tests__/bedrock.spec.ts index 645202486c..74f1977b2b 100644 --- a/src/api/providers/__tests__/bedrock.spec.ts +++ b/src/api/providers/__tests__/bedrock.spec.ts @@ -591,6 +591,76 @@ describe("AwsBedrockHandler", () => { }) }) + describe("prompt caching policy", () => { + function setupMockStreamText() { + async function* mockFullStream() { + yield { type: "text-delta", text: "Response" } + } + mockStreamText.mockReturnValue({ + fullStream: mockFullStream(), + usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }), + providerMetadata: Promise.resolve({}), + }) + } + + it("disables cache markers when global prompt caching is off", async () => { + setupMockStreamText() + const cacheDisabledHandler = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + promptCachingEnabled: false, + }) + + const generator = cacheDisabledHandler.createMessage("", [ + { + role: "user", + content: "Test prompt", + }, + ]) + for await (const _chunk of generator) { + // consume + } + + const callArgs = mockStreamText.mock.calls[0][0] + expect(callArgs.systemProviderOptions).toBeUndefined() + expect(callArgs.messages[0].providerOptions).toBeUndefined() + }) + + it("allows provider override to re-enable cache markers", async () => { + setupMockStreamText() + const overrideEnabledHandler = new AwsBedrockHandler({ + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsAccessKey: "test-access-key", + awsSecretKey: "test-secret-key", + awsRegion: "us-east-1", + promptCachingEnabled: false, + promptCachingProviderOverrides: { + bedrock: true, + }, + }) + + const generator = overrideEnabledHandler.createMessage("", [ + { + role: "user", + content: "Test prompt", + }, + ]) + for await (const _chunk of generator) { + // consume + } + + const callArgs = mockStreamText.mock.calls[0][0] + expect(callArgs.systemProviderOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + expect(callArgs.messages[0].providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + }) + }) + describe("error handling and validation", () => { it("should handle invalid regions gracefully", () => { expect(() => { diff --git a/src/api/providers/__tests__/minimax.spec.ts b/src/api/providers/__tests__/minimax.spec.ts index 3538184eee..60e7483ef8 100644 --- a/src/api/providers/__tests__/minimax.spec.ts +++ b/src/api/providers/__tests__/minimax.spec.ts @@ -366,6 +366,19 @@ describe("MiniMaxHandler", () => { }).rejects.toThrow("MiniMax: API Error") expect(mockHandleAiSdkError).toHaveBeenCalledWith(expect.any(Error), "MiniMax") }) + + it("disables prompt caching when globally disabled", async () => { + mockStreamText.mockReturnValue(createMockStream([{ type: "text-delta", text: "OK" }])) + + const handler = createHandler({ + promptCachingEnabled: false, + }) + await collectChunks(handler.createMessage(systemPrompt, messages)) + + const callArgs = mockStreamText.mock.calls[0]?.[0] + expect(callArgs.systemProviderOptions).toBeUndefined() + expect(callArgs.messages[0]?.providerOptions).toBeUndefined() + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/__tests__/openai-native.spec.ts b/src/api/providers/__tests__/openai-native.spec.ts index 568ed9ce97..34a88a0026 100644 --- a/src/api/providers/__tests__/openai-native.spec.ts +++ b/src/api/providers/__tests__/openai-native.spec.ts @@ -879,6 +879,63 @@ describe("OpenAiNativeHandler", () => { const callArgs = mockStreamText.mock.calls[0][0] expect(callArgs.providerOptions.openai.promptCacheRetention).toBeUndefined() }) + + it("should not pass promptCacheRetention when globally disabled", async () => { + async function* mockFullStream() { + yield { type: "text-delta", text: "Response" } + } + + mockStreamText.mockReturnValue({ + fullStream: mockFullStream(), + usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }), + providerMetadata: Promise.resolve({}), + content: Promise.resolve([]), + }) + + const h = new OpenAiNativeHandler({ + ...mockOptions, + apiModelId: "gpt-5.1", + promptCachingEnabled: false, + }) + + const stream = h.createMessage(systemPrompt, messages) + for await (const _ of stream) { + // consume + } + + const callArgs = mockStreamText.mock.calls[0][0] + expect(callArgs.providerOptions.openai.promptCacheRetention).toBeUndefined() + }) + + it("should pass promptCacheRetention when provider override enables it", async () => { + async function* mockFullStream() { + yield { type: "text-delta", text: "Response" } + } + + mockStreamText.mockReturnValue({ + fullStream: mockFullStream(), + usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }), + providerMetadata: Promise.resolve({}), + content: Promise.resolve([]), + }) + + const h = new OpenAiNativeHandler({ + ...mockOptions, + apiModelId: "gpt-5.1", + promptCachingEnabled: false, + promptCachingProviderOverrides: { + "openai-native": true, + }, + }) + + const stream = h.createMessage(systemPrompt, messages) + for await (const _ of stream) { + // consume + } + + const callArgs = mockStreamText.mock.calls[0][0] + expect(callArgs.providerOptions.openai.promptCacheRetention).toBe("24h") + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index 131b36992e..34c4c57624 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -19,13 +19,13 @@ import { shouldUseReasoningBudget } from "../../shared/api" import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { - convertToAiSdkMessages, convertToolsForAiSdk, processAiSdkStreamPart, mapToolChoice, handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" +import { applyPromptCacheToMessages } from "../transform/prompt-cache" import { calculateApiCostAnthropic } from "../../shared/cost" import { DEFAULT_HEADERS } from "./constants" @@ -119,45 +119,25 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple anthropicProviderOptions.disableParallelToolUse = true } - /** - * Vertex API has specific limitations for prompt caching: - * 1. Maximum of 4 blocks can have cache_control - * 2. Only text blocks can be cached (images and other content types cannot) - * 3. Cache control can only be applied to user messages, not assistant messages - * - * Our caching strategy: - * - Cache the system prompt (1 block) - * - Cache the last text block of the second-to-last user message (1 block) - * - Cache the last text block of the last user message (1 block) - * This ensures we stay under the 4-block limit while maintaining effective caching - * for the most relevant context. - */ - const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } } - - const userMsgIndices = messages.reduce( - (acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) - - const targetIndices = new Set() - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - - if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex) - if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex) - - if (targetIndices.size > 0) { - this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption) - } + const promptCache = applyPromptCacheToMessages({ + adapter: "anthropic", + overrideKey: "vertex", + messages: aiSdkMessages, + modelInfo: { + supportsPromptCache: modelConfig.info.supportsPromptCache, + promptCacheRetention: modelConfig.info.promptCacheRetention, + }, + settings: this.options, + }) // Build streamText request // Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values const requestOptions: Parameters[0] = { model: this.provider(modelConfig.id), system: systemPrompt, - ...({ - systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, - } as Record), + ...(promptCache.systemProviderOptions + ? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record) + : {}), messages: aiSdkMessages, temperature: modelConfig.temperature, maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -241,29 +221,6 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple } } - /** - * Apply cacheControl providerOptions to the correct AI SDK messages by walking - * the original Anthropic messages and converted AI SDK messages in parallel. - * - * convertToAiSdkMessages() can split a single Anthropic user message (containing - * tool_results + text) into 2 AI SDK messages (tool role + user role). This method - * accounts for that split so cache control lands on the right message. - */ - private applyCacheControlToAiSdkMessages( - aiSdkMessages: { role: string; providerOptions?: Record> }[], - targetIndices: Set, - cacheProviderOption: Record>, - ): void { - for (const idx of targetIndices) { - if (idx >= 0 && idx < aiSdkMessages.length) { - aiSdkMessages[idx].providerOptions = { - ...aiSdkMessages[idx].providerOptions, - ...cacheProviderOption, - } - } - } - } - getModel() { const modelId = this.options.apiModelId let id = modelId && modelId in vertexModels ? (modelId as VertexModelId) : vertexDefaultModelId diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 1f519250fa..03584f4115 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -17,13 +17,13 @@ import { shouldUseReasoningBudget } from "../../shared/api" import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { - convertToAiSdkMessages, convertToolsForAiSdk, processAiSdkStreamPart, mapToolChoice, handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" +import { applyPromptCacheToMessages } from "../transform/prompt-cache" import { calculateApiCostAnthropic } from "../../shared/cost" import { DEFAULT_HEADERS } from "./constants" @@ -105,34 +105,25 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa anthropicProviderOptions.disableParallelToolUse = true } - // Apply cache control to user messages - // Strategy: cache the last 2 user messages (write-to-cache + read-from-cache) - const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } } - - const userMsgIndices = messages.reduce( - (acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) - - const targetIndices = new Set() - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - - if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex) - if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex) - - if (targetIndices.size > 0) { - this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption) - } + const promptCache = applyPromptCacheToMessages({ + adapter: "anthropic", + overrideKey: "anthropic", + messages: aiSdkMessages, + modelInfo: { + supportsPromptCache: modelConfig.info.supportsPromptCache, + promptCacheRetention: modelConfig.info.promptCacheRetention, + }, + settings: this.options, + }) // Build streamText request // Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values const requestOptions: Parameters[0] = { model: this.provider(modelConfig.id), system: systemPrompt, - ...({ - systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, - } as Record), + ...(promptCache.systemProviderOptions + ? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record) + : {}), messages: aiSdkMessages, temperature: modelConfig.temperature, maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -216,29 +207,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa } } - /** - * Apply cacheControl providerOptions to the correct AI SDK messages by walking - * the original Anthropic messages and converted AI SDK messages in parallel. - * - * convertToAiSdkMessages() can split a single Anthropic user message (containing - * tool_results + text) into 2 AI SDK messages (tool role + user role). This method - * accounts for that split so cache control lands on the right message. - */ - private applyCacheControlToAiSdkMessages( - aiSdkMessages: { role: string; providerOptions?: Record> }[], - targetIndices: Set, - cacheProviderOption: Record>, - ): void { - for (const idx of targetIndices) { - if (idx >= 0 && idx < aiSdkMessages.length) { - aiSdkMessages[idx].providerOptions = { - ...aiSdkMessages[idx].providerOptions, - ...cacheProviderOption, - } - } - } - } - getModel() { const modelId = this.options.apiModelId let id = modelId && modelId in anthropicModels ? (modelId as AnthropicModelId) : anthropicDefaultModelId diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index bf713ea016..076449256f 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -25,7 +25,6 @@ import { TelemetryService } from "@roo-code/telemetry" import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { - convertToAiSdkMessages, convertToolsForAiSdk, processAiSdkStreamPart, mapToolChoice, @@ -33,6 +32,7 @@ import { yieldResponseMessage, } from "../transform/ai-sdk" import { getModelParams } from "../transform/model-params" +import { applyPromptCacheToMessages } from "../transform/prompt-cache" import { shouldUseReasoningBudget } from "../../shared/api" import { BaseProvider } from "./base-provider" import { DEFAULT_HEADERS } from "./constants" @@ -251,76 +251,25 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } } - // Prompt caching: use AI SDK's cachePoint mechanism - // The AI SDK's @ai-sdk/amazon-bedrock supports cachePoint in providerOptions per message. - // - // Strategy: Bedrock allows up to 4 cache checkpoints. We use them as: - // 1. System prompt (via systemProviderOptions below) - // 2-4. Up to 3 user messages in the conversation history - // - // For the message cache points, we target the last 2 user messages (matching - // Anthropic's strategy: write-to-cache + read-from-cache) PLUS an earlier "anchor" - // user message near the middle of the conversation. This anchor ensures the 20-block - // lookback window has a stable cache entry to hit, covering all assistant/tool messages - // between the anchor and the recent messages. - // - // We identify targets in the ORIGINAL Anthropic messages (before AI SDK conversion) - // because convertToAiSdkMessages() splits user messages containing tool_results into - // separate "tool" + "user" role messages, which would skew naive counting. - const usePromptCache = Boolean(this.options.awsUsePromptCache && this.supportsAwsPromptCache(modelConfig)) - - if (usePromptCache) { - const cachePointOption = { bedrock: { cachePoint: { type: "default" as const } } } - - // Find all user message indices in the original (pre-conversion) message array. - const originalUserIndices = filteredMessages.reduce( - (acc, msg, idx) => ("role" in msg && msg.role === "user" ? [...acc, idx] : acc), - [], - ) - - // Select up to 3 user messages for cache points (system prompt uses the 4th): - // - Last user message: write to cache for next request - // - Second-to-last user message: read from cache for current request - // - An "anchor" message earlier in the conversation for 20-block window coverage - const targetOriginalIndices = new Set() - const numUserMsgs = originalUserIndices.length - - if (numUserMsgs >= 1) { - // Always cache the last user message - targetOriginalIndices.add(originalUserIndices[numUserMsgs - 1]) - } - if (numUserMsgs >= 2) { - // Cache the second-to-last user message - targetOriginalIndices.add(originalUserIndices[numUserMsgs - 2]) - } - if (numUserMsgs >= 5) { - // Add an anchor cache point roughly in the first third of user messages. - // This ensures that the 20-block lookback from the second-to-last breakpoint - // can find a stable cache entry, covering all the assistant and tool messages - // in the middle of the conversation. We pick the user message at ~1/3 position. - const anchorIdx = Math.floor(numUserMsgs / 3) - // Only add if it's not already one of the last-2 targets - if (!targetOriginalIndices.has(originalUserIndices[anchorIdx])) { - targetOriginalIndices.add(originalUserIndices[anchorIdx]) - } - } - - // Apply cachePoint to the correct AI SDK messages by walking both arrays in parallel. - // A single original user message with tool_results becomes [tool-role msg, user-role msg] - // in the AI SDK array, while a plain user message becomes [user-role msg]. - if (targetOriginalIndices.size > 0) { - this.applyCachePointsToAiSdkMessages(aiSdkMessages, targetOriginalIndices, cachePointOption) - } - } + const promptCache = applyPromptCacheToMessages({ + adapter: "bedrock", + overrideKey: "bedrock", + messages: aiSdkMessages, + modelInfo: { + supportsPromptCache: modelConfig.info.supportsPromptCache, + promptCacheRetention: modelConfig.info.promptCacheRetention, + }, + settings: this.options, + }) // Build streamText request // Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values const requestOptions: Parameters[0] = { model: this.provider(modelConfig.id), system: systemPrompt, - ...(usePromptCache && { - systemProviderOptions: { bedrock: { cachePoint: { type: "default" } } } as Record, - }), + ...(promptCache.systemProviderOptions + ? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record) + : {}), messages: aiSdkMessages, temperature: modelConfig.temperature ?? (this.options.modelTemperature as number), maxOutputTokens: modelConfig.maxTokens || (modelConfig.info.maxTokens as number), @@ -692,43 +641,6 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH } } - /************************************************************************************ - * - * CACHE - * - *************************************************************************************/ - - private supportsAwsPromptCache(modelConfig: { id: BedrockModelId | string; info: ModelInfo }): boolean | undefined { - return ( - modelConfig?.info?.supportsPromptCache && - (modelConfig?.info as any)?.cachableFields && - (modelConfig?.info as any)?.cachableFields?.length > 0 - ) - } - - /** - * Apply cachePoint providerOptions to the correct AI SDK messages by walking - * the original Anthropic messages and converted AI SDK messages in parallel. - * - * convertToAiSdkMessages() can split a single Anthropic user message (containing - * tool_results + text) into 2 AI SDK messages (tool role + user role). This method - * accounts for that split so cache points land on the right message. - */ - private applyCachePointsToAiSdkMessages( - aiSdkMessages: { role: string; providerOptions?: Record> }[], - targetIndices: Set, - cachePointOption: Record>, - ): void { - for (const idx of targetIndices) { - if (idx >= 0 && idx < aiSdkMessages.length) { - aiSdkMessages[idx].providerOptions = { - ...aiSdkMessages[idx].providerOptions, - ...cachePointOption, - } - } - } - } - /************************************************************************************ * * AMAZON REGIONS diff --git a/src/api/providers/minimax.ts b/src/api/providers/minimax.ts index 17b0055e4e..e5a83b7a8e 100644 --- a/src/api/providers/minimax.ts +++ b/src/api/providers/minimax.ts @@ -9,13 +9,13 @@ import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" import { mergeEnvironmentDetailsForMiniMax } from "../transform/minimax-format" import { - convertToAiSdkMessages, convertToolsForAiSdk, processAiSdkStreamPart, mapToolChoice, handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" +import { applyPromptCacheToMessages } from "../transform/prompt-cache" import { calculateApiCostAnthropic } from "../../shared/cost" import { DEFAULT_HEADERS } from "./constants" @@ -72,7 +72,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand }) const mergedMessages = mergeEnvironmentDetailsForMiniMax(messages as any) - const aiSdkMessages = mergedMessages as ModelMessage[] + const aiSdkMessages = (mergedMessages as ModelMessage[]).map((message) => ({ ...message })) const openAiTools = this.convertToolsForOpenAI(metadata?.tools) const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined @@ -89,29 +89,23 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand anthropicProviderOptions.disableParallelToolUse = true } - const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } } - const userMsgIndices = mergedMessages.reduce( - (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) - - const targetIndices = new Set() - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - - if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex) - if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex) - - if (targetIndices.size > 0) { - this.applyCacheControlToAiSdkMessages(aiSdkMessages, targetIndices, cacheProviderOption) - } + const promptCache = applyPromptCacheToMessages({ + adapter: "anthropic", + overrideKey: "minimax", + messages: aiSdkMessages, + modelInfo: { + supportsPromptCache: modelConfig.info.supportsPromptCache, + promptCacheRetention: (modelConfig.info as ModelInfo).promptCacheRetention, + }, + settings: this.options, + }) const requestOptions = { model: this.client(modelConfig.id), system: systemPrompt, - ...({ - systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, - } as Record), + ...(promptCache.systemProviderOptions + ? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record) + : {}), messages: aiSdkMessages, temperature: modelParams.temperature, maxOutputTokens: modelParams.maxTokens ?? modelConfig.info.maxTokens, @@ -187,21 +181,6 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand } } - private applyCacheControlToAiSdkMessages( - aiSdkMessages: { role: string; providerOptions?: Record> }[], - targetIndices: Set, - cacheProviderOption: Record>, - ): void { - for (const idx of targetIndices) { - if (idx >= 0 && idx < aiSdkMessages.length) { - aiSdkMessages[idx].providerOptions = { - ...aiSdkMessages[idx].providerOptions, - ...cacheProviderOption, - } - } - } - } - getModel() { const modelId = this.options.apiModelId diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index c966cc3537..b99103a857 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -19,13 +19,8 @@ import { import type { ApiHandlerOptions } from "../../shared/api" import { calculateApiCostOpenAI } from "../../shared/cost" -import { - convertToAiSdkMessages, - convertToolsForAiSdk, - consumeAiSdkStream, - mapToolChoice, - handleAiSdkError, -} from "../transform/ai-sdk" +import { convertToolsForAiSdk, consumeAiSdkStream, mapToolChoice, handleAiSdkError } from "../transform/ai-sdk" +import { applyPromptCacheToMessages } from "../transform/prompt-cache" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" @@ -265,15 +260,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio return selected && selected !== "disable" ? (selected as any) : undefined } - /** - * Returns the appropriate prompt cache retention policy for the given model, if any. - */ - private getPromptCacheRetention(model: OpenAiNativeModel): "24h" | undefined { - if (!model.info.supportsPromptCache) return undefined - if (model.info.promptCacheRetention === "24h") return "24h" - return undefined - } - /** * Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available. */ @@ -301,7 +287,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio systemPrompt?: string, ): Record { const reasoningEffort = this.getReasoningEffort(model) - const promptCacheRetention = this.getPromptCacheRetention(model) + const promptCache = applyPromptCacheToMessages({ + adapter: "openai-native", + overrideKey: "openai-native", + messages: [], + modelInfo: { + supportsPromptCache: model.info.supportsPromptCache, + promptCacheRetention: model.info.promptCacheRetention, + }, + settings: this.options, + }) const requestedTier = (this.options.openAiNativeServiceTier as ServiceTier | undefined) || undefined const allowedTierNames = new Set(model.info.tiers?.map((t) => t.name).filter(Boolean) || []) @@ -329,8 +324,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio openaiOptions.serviceTier = requestedTier } - if (promptCacheRetention) { - openaiOptions.promptCacheRetention = promptCacheRetention + if (promptCache.providerOptionsPatch?.openai?.promptCacheRetention) { + openaiOptions.promptCacheRetention = promptCache.providerOptionsPatch.openai.promptCacheRetention } return { openai: openaiOptions } diff --git a/src/api/transform/__tests__/prompt-cache.spec.ts b/src/api/transform/__tests__/prompt-cache.spec.ts new file mode 100644 index 0000000000..d8724f454a --- /dev/null +++ b/src/api/transform/__tests__/prompt-cache.spec.ts @@ -0,0 +1,210 @@ +import type { ModelMessage } from "ai" + +import { applyPromptCacheToMessages, resolvePromptCachePolicy } from "../prompt-cache" + +describe("prompt-cache", () => { + describe("resolvePromptCachePolicy", () => { + it("defaults to enabled with aggressive strategy", () => { + const policy = resolvePromptCachePolicy({ + overrideKey: "bedrock", + supportsPromptCache: true, + }) + + expect(policy).toEqual({ + enabled: true, + strategy: "aggressive", + }) + }) + + it("uses provider override over global setting", () => { + const disabledByGlobal = resolvePromptCachePolicy({ + overrideKey: "bedrock", + supportsPromptCache: true, + settings: { + promptCachingEnabled: false, + }, + }) + + expect(disabledByGlobal.enabled).toBe(false) + + const enabledByOverride = resolvePromptCachePolicy({ + overrideKey: "bedrock", + supportsPromptCache: true, + settings: { + promptCachingEnabled: false, + promptCachingProviderOverrides: { + bedrock: true, + }, + }, + }) + + expect(enabledByOverride.enabled).toBe(true) + + const disabledByOverride = resolvePromptCachePolicy({ + overrideKey: "bedrock", + supportsPromptCache: true, + settings: { + promptCachingEnabled: true, + promptCachingProviderOverrides: { + bedrock: false, + }, + }, + }) + + expect(disabledByOverride.enabled).toBe(false) + }) + + it("disables caching for unsupported models", () => { + const policy = resolvePromptCachePolicy({ + overrideKey: "anthropic", + supportsPromptCache: false, + settings: { + promptCachingEnabled: true, + }, + }) + + expect(policy.enabled).toBe(false) + }) + }) + + describe("applyPromptCacheToMessages", () => { + function buildMessages(): ModelMessage[] { + return [ + { + role: "user", + content: [{ type: "text", text: "u1" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "a1" }], + }, + { + role: "user", + content: [{ type: "text", text: "u2" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "a2" }], + }, + { + role: "user", + content: [{ type: "text", text: "u3" }], + }, + ] + } + + it("applies anthropic strategy and system marker", () => { + const messages = buildMessages() + const result = applyPromptCacheToMessages({ + adapter: "anthropic", + overrideKey: "anthropic", + messages, + modelInfo: { + supportsPromptCache: true, + }, + settings: { + promptCachingStrategy: "aggressive", + }, + }) + + expect(result.systemProviderOptions).toEqual({ + anthropic: { cacheControl: { type: "ephemeral" } }, + }) + expect((messages[0] as any).providerOptions).toBeUndefined() + expect((messages[2] as any).providerOptions).toEqual({ + anthropic: { cacheControl: { type: "ephemeral" } }, + }) + expect((messages[4] as any).providerOptions).toEqual({ + anthropic: { cacheControl: { type: "ephemeral" } }, + }) + }) + + it("applies bedrock aggressive checkpoints to last three user messages", () => { + const messages = buildMessages() + const result = applyPromptCacheToMessages({ + adapter: "bedrock", + overrideKey: "bedrock", + messages, + modelInfo: { + supportsPromptCache: true, + }, + settings: { + promptCachingStrategy: "aggressive", + }, + }) + + expect(result.systemProviderOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + expect((messages[0] as any).providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + expect((messages[2] as any).providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + expect((messages[4] as any).providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + }) + + it("applies balanced strategy with fewer checkpoints", () => { + const messages = buildMessages() + applyPromptCacheToMessages({ + adapter: "bedrock", + overrideKey: "bedrock", + messages, + modelInfo: { + supportsPromptCache: true, + }, + settings: { + promptCachingStrategy: "balanced", + }, + }) + + expect((messages[0] as any).providerOptions).toBeUndefined() + expect((messages[2] as any).providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + expect((messages[4] as any).providerOptions).toEqual({ + bedrock: { cachePoint: { type: "default" } }, + }) + }) + + it("returns openai retention patch when enabled", () => { + const messages: ModelMessage[] = [] + const result = applyPromptCacheToMessages({ + adapter: "openai-native", + overrideKey: "openai-native", + messages, + modelInfo: { + supportsPromptCache: true, + promptCacheRetention: "24h", + }, + }) + + expect(result.providerOptionsPatch).toEqual({ + openai: { + promptCacheRetention: "24h", + }, + }) + }) + + it("does not return openai retention patch when globally disabled", () => { + const result = applyPromptCacheToMessages({ + adapter: "openai-native", + overrideKey: "openai-native", + messages: [], + modelInfo: { + supportsPromptCache: true, + promptCacheRetention: "24h", + }, + settings: { + promptCachingEnabled: false, + }, + }) + + expect(result.enabled).toBe(false) + expect(result.providerOptionsPatch).toBeUndefined() + }) + }) +}) diff --git a/src/api/transform/prompt-cache.ts b/src/api/transform/prompt-cache.ts new file mode 100644 index 0000000000..6635657bd1 --- /dev/null +++ b/src/api/transform/prompt-cache.ts @@ -0,0 +1,175 @@ +import type { ModelInfo, ProviderSettings } from "@roo-code/types" +import type { ModelMessage } from "ai" + +export type PromptCachingStrategy = NonNullable +export type PromptCacheAdapter = "anthropic" | "bedrock" | "openai-native" + +export interface PromptCachePolicy { + enabled: boolean + strategy: PromptCachingStrategy +} + +export interface ApplyPromptCacheArgs { + adapter: PromptCacheAdapter + overrideKey: string + messages: ModelMessage[] + modelInfo: Pick + settings?: Pick< + ProviderSettings, + "promptCachingEnabled" | "promptCachingStrategy" | "promptCachingProviderOverrides" + > +} + +export interface AppliedPromptCache { + enabled: boolean + strategy: PromptCachingStrategy + systemProviderOptions?: Record + providerOptionsPatch?: Record> +} + +const DEFAULT_PROMPT_CACHING_STRATEGY: PromptCachingStrategy = "aggressive" + +export function resolvePromptCachePolicy({ + overrideKey, + settings, + supportsPromptCache, +}: { + overrideKey: string + settings?: Pick< + ProviderSettings, + "promptCachingEnabled" | "promptCachingStrategy" | "promptCachingProviderOverrides" + > + supportsPromptCache: boolean +}): PromptCachePolicy { + const strategy = settings?.promptCachingStrategy ?? DEFAULT_PROMPT_CACHING_STRATEGY + if (!supportsPromptCache) { + return { enabled: false, strategy } + } + + const globalEnabled = settings?.promptCachingEnabled ?? true + const providerOverride = settings?.promptCachingProviderOverrides?.[overrideKey] + const enabled = providerOverride ?? globalEnabled + + return { enabled, strategy } +} + +export function applyPromptCacheToMessages({ + adapter, + overrideKey, + messages, + modelInfo, + settings, +}: ApplyPromptCacheArgs): AppliedPromptCache { + const policy = resolvePromptCachePolicy({ + overrideKey, + settings, + supportsPromptCache: modelInfo.supportsPromptCache, + }) + + if (!policy.enabled) { + return { + enabled: false, + strategy: policy.strategy, + } + } + + if (adapter === "openai-native") { + if (modelInfo.promptCacheRetention === "24h") { + return { + enabled: true, + strategy: policy.strategy, + providerOptionsPatch: { + openai: { + promptCacheRetention: "24h", + }, + }, + } + } + + return { + enabled: true, + strategy: policy.strategy, + } + } + + const adapterConfig = getMessageAdapterConfig(adapter) + const checkpointCount = resolveCheckpointCount(policy.strategy, adapterConfig.maxUserCheckpoints) + const userIndices = getUserMessageIndices(messages) + const targetIndices = userIndices.slice(-checkpointCount) + + applyProviderOptionAtIndices(messages, targetIndices, adapterConfig.messageProviderOption) + + return { + enabled: true, + strategy: policy.strategy, + systemProviderOptions: adapterConfig.systemProviderOptions, + } +} + +function getMessageAdapterConfig(adapter: Exclude): { + maxUserCheckpoints: number + systemProviderOptions: Record + messageProviderOption: Record> +} { + if (adapter === "bedrock") { + return { + maxUserCheckpoints: 3, + systemProviderOptions: { + bedrock: { cachePoint: { type: "default" } }, + }, + messageProviderOption: { + bedrock: { cachePoint: { type: "default" } }, + }, + } + } + + return { + maxUserCheckpoints: 2, + systemProviderOptions: { + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + messageProviderOption: { + anthropic: { cacheControl: { type: "ephemeral" } }, + }, + } +} + +function resolveCheckpointCount(strategy: PromptCachingStrategy, maxUserCheckpoints: number): number { + if (maxUserCheckpoints <= 0) { + return 0 + } + + if (strategy === "conservative") { + return 1 + } + + if (strategy === "balanced") { + return Math.max(1, Math.ceil(maxUserCheckpoints / 2)) + } + + return maxUserCheckpoints +} + +function getUserMessageIndices(messages: ModelMessage[]): number[] { + const indices: number[] = [] + for (let i = 0; i < messages.length; i++) { + if (messages[i].role === "user") { + indices.push(i) + } + } + return indices +} + +function applyProviderOptionAtIndices( + messages: ModelMessage[], + indices: number[], + providerOption: Record>, +): void { + for (const index of indices) { + const message = messages[index] as ModelMessage & { providerOptions?: unknown } + message.providerOptions = { + ...((message.providerOptions as Record | undefined) ?? {}), + ...providerOption, + } as any + } +} diff --git a/src/core/config/ContextProxy.ts b/src/core/config/ContextProxy.ts index 2825d1c945..588bcf250a 100644 --- a/src/core/config/ContextProxy.ts +++ b/src/core/config/ContextProxy.ts @@ -22,6 +22,7 @@ import { TelemetryService } from "@roo-code/telemetry" import { logger } from "../../utils/logging" import { supportPrompt } from "../../shared/support-prompt" +import { migrateLegacyPromptCacheSettings } from "./migrateLegacyPromptCacheSettings" type GlobalStateKey = keyof GlobalState type SecretStateKey = keyof SecretState @@ -94,6 +95,9 @@ export class ContextProxy { // Migration: Sanitize invalid/removed API providers await this.migrateInvalidApiProvider() + // Migration: one-time read of removed provider-level prompt cache keys. + await this.migrateLegacyPromptCacheKeys() + // Migration: Move legacy customCondensingPrompt to customSupportPrompts await this.migrateLegacyCondensingPrompt() @@ -103,6 +107,45 @@ export class ContextProxy { this._isInitialized = true } + /** + * Migrates removed provider-level prompt cache toggles to the unified + * `promptCachingProviderOverrides` map and clears the legacy keys. + */ + private async migrateLegacyPromptCacheKeys() { + try { + const rawAwsToggle = this.originalContext.globalState.get("awsUsePromptCache" as any) + const rawLiteLlmToggle = this.originalContext.globalState.get("litellmUsePromptCache" as any) + + if (rawAwsToggle === undefined && rawLiteLlmToggle === undefined) { + return + } + + const migrationInput: Record = { + ...this.stateCache, + awsUsePromptCache: rawAwsToggle, + litellmUsePromptCache: rawLiteLlmToggle, + } + + const migration = migrateLegacyPromptCacheSettings(migrationInput) + if (!migration.changed) { + return + } + + const overrides = migration.config.promptCachingProviderOverrides as Record | undefined + await this.originalContext.globalState.update("promptCachingProviderOverrides", overrides) + this.stateCache.promptCachingProviderOverrides = overrides + + await this.originalContext.globalState.update("awsUsePromptCache" as any, undefined) + await this.originalContext.globalState.update("litellmUsePromptCache" as any, undefined) + delete (this.stateCache as Record).awsUsePromptCache + delete (this.stateCache as Record).litellmUsePromptCache + } catch (error) { + logger.error( + `Error during legacy prompt cache migration: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + /** * Migrates the legacy customCondensingPrompt to the new customSupportPrompts structure * and removes the legacy field. @@ -459,6 +502,13 @@ export class ContextProxy { } } + const promptCacheMigration = migrateLegacyPromptCacheSettings({ + ...(sanitizedValues as unknown as Record), + awsUsePromptCache: (this.stateCache as Record).awsUsePromptCache, + litellmUsePromptCache: (this.stateCache as Record).litellmUsePromptCache, + }) + sanitizedValues = promptCacheMigration.config as RooCodeSettings + const isKnownProvider = typeof values.apiProvider === "string" && (isProviderName(values.apiProvider) || isRetiredProvider(values.apiProvider)) diff --git a/src/core/config/ProviderSettingsManager.ts b/src/core/config/ProviderSettingsManager.ts index 6088bd68fe..7fdf55cebd 100644 --- a/src/core/config/ProviderSettingsManager.ts +++ b/src/core/config/ProviderSettingsManager.ts @@ -18,6 +18,7 @@ import { TelemetryService } from "@roo-code/telemetry" import { Mode, modes } from "../../shared/modes" import { buildApiHandler } from "../../api" +import { migrateLegacyPromptCacheSettings } from "./migrateLegacyPromptCacheSettings" // Type-safe model migrations mapping type ModelMigrations = { @@ -128,6 +129,10 @@ export class ProviderSettingsManager { isDirty = true } + if (this.applyLegacyPromptCacheMigration(providerProfiles)) { + isDirty = true + } + // Ensure all configs have IDs. for (const [_name, apiConfig] of Object.entries(providerProfiles.apiConfigs)) { if (!apiConfig.id) { @@ -313,6 +318,22 @@ export class ProviderSettingsManager { return migrated } + private applyLegacyPromptCacheMigration(providerProfiles: ProviderProfiles): boolean { + let migrated = false + + for (const [name, apiConfig] of Object.entries(providerProfiles.apiConfigs)) { + const migrationResult = migrateLegacyPromptCacheSettings(apiConfig as unknown as Record) + if (!migrationResult.changed) { + continue + } + + providerProfiles.apiConfigs[name] = migrationResult.config as ProviderSettingsWithId + migrated = true + } + + return migrated + } + /** * Clean model ID by removing prefix before "/" */ @@ -645,7 +666,8 @@ export class ProviderSettingsManager { return apiConfig } - const config = apiConfig as Record + const migrationResult = migrateLegacyPromptCacheSettings(apiConfig as Record) + const config = migrationResult.config as Record const apiProvider = config.apiProvider @@ -663,7 +685,7 @@ export class ProviderSettingsManager { return restConfig } - return apiConfig + return config } private async store(providerProfiles: ProviderProfiles) { diff --git a/src/core/config/__tests__/ContextProxy.spec.ts b/src/core/config/__tests__/ContextProxy.spec.ts index 7c1d2a6e3c..f11d80c111 100644 --- a/src/core/config/__tests__/ContextProxy.spec.ts +++ b/src/core/config/__tests__/ContextProxy.spec.ts @@ -70,16 +70,20 @@ describe("ContextProxy", () => { describe("constructor", () => { it("should initialize state cache with all global state keys", () => { - // +3 for the migration checks: + // +5 for the migration checks: // 1. openRouterImageGenerationSettings - // 2. customCondensingPrompt - // 3. customSupportPrompts (for migrateOldDefaultCondensingPrompt) - expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 3) + // 2. awsUsePromptCache + // 3. litellmUsePromptCache + // 4. customCondensingPrompt + // 5. customSupportPrompts (for migrateOldDefaultCondensingPrompt) + expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 5) for (const key of GLOBAL_STATE_KEYS) { expect(mockGlobalState.get).toHaveBeenCalledWith(key) } // Also check for migration calls expect(mockGlobalState.get).toHaveBeenCalledWith("openRouterImageGenerationSettings") + expect(mockGlobalState.get).toHaveBeenCalledWith("awsUsePromptCache") + expect(mockGlobalState.get).toHaveBeenCalledWith("litellmUsePromptCache") expect(mockGlobalState.get).toHaveBeenCalledWith("customCondensingPrompt") expect(mockGlobalState.get).toHaveBeenCalledWith("customSupportPrompts") }) @@ -104,8 +108,8 @@ describe("ContextProxy", () => { const result = proxy.getGlobalState("apiProvider") expect(result).toBe("deepseek") - // Original context should be called once during updateGlobalState (+3 for migration checks) - expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 3) // From initialization + migration checks + // Original context should be called once during updateGlobalState (+5 for migration checks) + expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 5) // From initialization + migration checks }) it("should handle default values correctly", async () => { @@ -553,6 +557,23 @@ describe("ContextProxy", () => { // Should not throw and should return undefined expect(settings.apiProvider).toBeUndefined() }) + + it("should migrate legacy prompt cache toggles to provider overrides", async () => { + await proxy.setValues({ + apiProvider: "bedrock", + awsUsePromptCache: false as any, + litellmUsePromptCache: false as any, + } as any) + + const settings = proxy.getProviderSettings() + + expect((settings as any).awsUsePromptCache).toBeUndefined() + expect((settings as any).litellmUsePromptCache).toBeUndefined() + expect(settings.promptCachingProviderOverrides).toEqual({ + bedrock: false, + litellm: false, + }) + }) }) describe("old default condensing prompt migration", () => { diff --git a/src/core/config/__tests__/ProviderSettingsManager.spec.ts b/src/core/config/__tests__/ProviderSettingsManager.spec.ts index 3f6b4f7847..1fe891a69b 100644 --- a/src/core/config/__tests__/ProviderSettingsManager.spec.ts +++ b/src/core/config/__tests__/ProviderSettingsManager.spec.ts @@ -336,6 +336,45 @@ describe("ProviderSettingsManager", () => { expect(storedConfig.apiConfigs.default.apiModelId).toEqual("roo/code-supernova-1-million") }) + it("should migrate legacy prompt cache toggles to provider overrides", async () => { + mockSecrets.get.mockResolvedValue( + JSON.stringify({ + currentApiConfigName: "default", + apiConfigs: { + default: { + id: "default", + apiProvider: "bedrock", + awsUsePromptCache: false, + }, + lite: { + id: "lite", + apiProvider: "litellm", + litellmUsePromptCache: false, + }, + }, + migrations: { + rateLimitSecondsMigrated: true, + openAiHeadersMigrated: true, + consecutiveMistakeLimitMigrated: true, + todoListEnabledMigrated: true, + claudeCodeLegacySettingsMigrated: true, + }, + }), + ) + + await providerSettingsManager.initialize() + + expect(mockSecrets.store).toHaveBeenCalled() + const calls = mockSecrets.store.mock.calls + const storedConfig = JSON.parse(calls[calls.length - 1][1]) + + expect(storedConfig.apiConfigs.default.awsUsePromptCache).toBeUndefined() + expect(storedConfig.apiConfigs.default.promptCachingProviderOverrides).toEqual({ bedrock: false }) + + expect(storedConfig.apiConfigs.lite.litellmUsePromptCache).toBeUndefined() + expect(storedConfig.apiConfigs.lite.promptCachingProviderOverrides).toEqual({ litellm: false }) + }) + it("should throw error if secrets storage fails", async () => { mockSecrets.get.mockRejectedValue(new Error("Storage failed")) diff --git a/src/core/config/__tests__/migrateLegacyPromptCacheSettings.spec.ts b/src/core/config/__tests__/migrateLegacyPromptCacheSettings.spec.ts new file mode 100644 index 0000000000..58f0ba8277 --- /dev/null +++ b/src/core/config/__tests__/migrateLegacyPromptCacheSettings.spec.ts @@ -0,0 +1,67 @@ +import { migrateLegacyPromptCacheSettings } from "../migrateLegacyPromptCacheSettings" + +describe("migrateLegacyPromptCacheSettings", () => { + it("maps legacy false toggles to provider overrides and removes legacy keys", () => { + const input = { + apiProvider: "bedrock", + awsUsePromptCache: false, + litellmUsePromptCache: false, + } + + const result = migrateLegacyPromptCacheSettings(input) + + expect(result.changed).toBe(true) + expect(result.config).toEqual({ + apiProvider: "bedrock", + promptCachingProviderOverrides: { + bedrock: false, + litellm: false, + }, + }) + }) + + it("does not create overrides for legacy true toggles and still removes legacy keys", () => { + const input = { + apiProvider: "bedrock", + awsUsePromptCache: true, + litellmUsePromptCache: true, + } + + const result = migrateLegacyPromptCacheSettings(input) + + expect(result.changed).toBe(true) + expect(result.config).toEqual({ + apiProvider: "bedrock", + }) + }) + + it("does not overwrite explicit new-format overrides", () => { + const input = { + awsUsePromptCache: false, + promptCachingProviderOverrides: { + bedrock: true, + }, + } + + const result = migrateLegacyPromptCacheSettings(input) + + expect(result.changed).toBe(true) + expect(result.config).toEqual({ + promptCachingProviderOverrides: { + bedrock: true, + }, + }) + }) + + it("returns unchanged when no legacy keys exist", () => { + const input = { + apiProvider: "anthropic", + promptCachingEnabled: true, + } + + const result = migrateLegacyPromptCacheSettings(input) + + expect(result.changed).toBe(false) + expect(result.config).toEqual(input) + }) +}) diff --git a/src/core/config/migrateLegacyPromptCacheSettings.ts b/src/core/config/migrateLegacyPromptCacheSettings.ts new file mode 100644 index 0000000000..b722ffce43 --- /dev/null +++ b/src/core/config/migrateLegacyPromptCacheSettings.ts @@ -0,0 +1,50 @@ +type PromptCacheMigrationInput = Record + +export interface PromptCacheMigrationResult { + config: T + changed: boolean +} + +/** + * One-time migration helper for legacy provider-specific prompt cache toggles. + * - Maps legacy `false` values to provider overrides + * - Drops legacy keys from the config object + */ +export function migrateLegacyPromptCacheSettings( + config: T, +): PromptCacheMigrationResult { + let changed = false + const next = { ...config } as Record + + const currentOverrides = next.promptCachingProviderOverrides + const overrides = + typeof currentOverrides === "object" && currentOverrides !== null && !Array.isArray(currentOverrides) + ? { ...(currentOverrides as Record) } + : {} + + if (next.awsUsePromptCache === false && overrides.bedrock === undefined) { + overrides.bedrock = false + changed = true + } + + if (next.litellmUsePromptCache === false && overrides.litellm === undefined) { + overrides.litellm = false + changed = true + } + + if (Object.keys(overrides).length > 0) { + next.promptCachingProviderOverrides = overrides + } + + if ("awsUsePromptCache" in next) { + delete next.awsUsePromptCache + changed = true + } + + if ("litellmUsePromptCache" in next) { + delete next.litellmUsePromptCache + changed = true + } + + return { config: next as T, changed } +} diff --git a/webview-ui/src/components/settings/ApiOptions.tsx b/webview-ui/src/components/settings/ApiOptions.tsx index 51210de4f4..1725778021 100644 --- a/webview-ui/src/components/settings/ApiOptions.tsx +++ b/webview-ui/src/components/settings/ApiOptions.tsx @@ -3,6 +3,7 @@ import { convertHeadersToObject } from "./utils/headers" import { useDebounce } from "react-use" import { VSCodeLink } from "@vscode/webview-ui-toolkit/react" import { ExternalLinkIcon } from "@radix-ui/react-icons" +import { Checkbox } from "vscrui" import { type ProviderName, @@ -573,7 +574,6 @@ const ApiOptions = ({ )} @@ -794,6 +794,40 @@ const ApiOptions = ({ } onChange={(value) => setApiConfigurationField("consecutiveMistakeLimit", value)} /> + + setApiConfigurationField("promptCachingEnabled", checked) + }> +
+ {t("settings:providers.enablePromptCaching")} +
+ +
+ {t("settings:providers.enablePromptCachingTitle")} +
+ {(apiConfiguration.promptCachingEnabled ?? true) && ( +
+ + +
+ )} {selectedProvider === "openrouter" && openRouterModelProviders && Object.keys(openRouterModelProviders).length > 0 && ( diff --git a/webview-ui/src/components/settings/__tests__/ApiOptions.spec.tsx b/webview-ui/src/components/settings/__tests__/ApiOptions.spec.tsx index 469dcd914b..3464fd424a 100644 --- a/webview-ui/src/components/settings/__tests__/ApiOptions.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/ApiOptions.spec.tsx @@ -304,6 +304,43 @@ describe("ApiOptions", () => { expect(screen.getByTestId("rate-limit-seconds-control")).toBeInTheDocument() }) + it("shows global prompt caching controls in advanced settings", () => { + renderApiOptions({ + apiConfiguration: {}, + }) + + expect(screen.getByText("settings:providers.enablePromptCaching")).toBeInTheDocument() + expect(screen.getByText("Prompt caching strategy")).toBeInTheDocument() + }) + + it("updates prompt caching fields from advanced settings controls", () => { + const mockSetApiConfigurationField = vi.fn() + renderApiOptions({ + apiConfiguration: {}, + setApiConfigurationField: mockSetApiConfigurationField, + }) + + const enablePromptCachingLabel = screen.getByText("settings:providers.enablePromptCaching").closest("label") + const enablePromptCachingInput = enablePromptCachingLabel?.querySelector("input") as HTMLInputElement + fireEvent.click(enablePromptCachingInput) + expect(mockSetApiConfigurationField).toHaveBeenCalledWith("promptCachingEnabled", false) + + const conservativeOption = screen.getByText("Conservative") + const strategySelect = conservativeOption.closest("select") as HTMLSelectElement + fireEvent.change(strategySelect, { target: { value: "conservative" } }) + expect(mockSetApiConfigurationField).toHaveBeenCalledWith("promptCachingStrategy", expect.any(String)) + }) + + it("hides prompt caching strategy selector when prompt caching is disabled", () => { + renderApiOptions({ + apiConfiguration: { + promptCachingEnabled: false, + }, + }) + + expect(screen.queryByText("Prompt caching strategy")).not.toBeInTheDocument() + }) + it("hides all controls when fromWelcomeView is true", () => { renderApiOptions({ fromWelcomeView: true }) expect(screen.queryByTestId("temperature-control")).not.toBeInTheDocument() diff --git a/webview-ui/src/components/settings/providers/Bedrock.tsx b/webview-ui/src/components/settings/providers/Bedrock.tsx index d9c69f8a8e..ea0d50783f 100644 --- a/webview-ui/src/components/settings/providers/Bedrock.tsx +++ b/webview-ui/src/components/settings/providers/Bedrock.tsx @@ -4,7 +4,6 @@ import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react" import { type ProviderSettings, - type ModelInfo, type BedrockServiceTier, BEDROCK_REGIONS, BEDROCK_1M_CONTEXT_MODEL_IDS, @@ -13,18 +12,17 @@ import { } from "@roo-code/types" import { useAppTranslation } from "@src/i18n/TranslationContext" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, StandardTooltip } from "@src/components/ui" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@src/components/ui" -import { inputEventTransform, noTransform } from "../transforms" +import { inputEventTransform } from "../transforms" type BedrockProps = { apiConfiguration: ProviderSettings setApiConfigurationField: (field: keyof ProviderSettings, value: ProviderSettings[keyof ProviderSettings]) => void - selectedModelInfo?: ModelInfo simplifySettings?: boolean } -export const Bedrock = ({ apiConfiguration, setApiConfigurationField, selectedModelInfo }: BedrockProps) => { +export const Bedrock = ({ apiConfiguration, setApiConfigurationField }: BedrockProps) => { const { t } = useAppTranslation() const [awsEndpointSelected, setAwsEndpointSelected] = useState(!!apiConfiguration?.awsBedrockEndpointEnabled) @@ -195,26 +193,6 @@ export const Bedrock = ({ apiConfiguration, setApiConfigurationField, selectedMo }}> {t("settings:providers.awsCrossRegion")} - {selectedModelInfo?.supportsPromptCache && ( - <> - -
- {t("settings:providers.enablePromptCaching")} - - - -
- -
- {t("settings:providers.cacheUsageNote")} -
- - )} {supports1MContextBeta && (
- - {/* Show prompt caching option if the selected model supports it */} - {(() => { - const selectedModelId = apiConfiguration.litellmModelId || litellmDefaultModelId - const selectedModel = routerModels?.litellm?.[selectedModelId] - if (selectedModel?.supportsPromptCache) { - return ( -
- { - setApiConfigurationField("litellmUsePromptCache", e.target.checked) - }}> - {t("settings:providers.enablePromptCaching")} - -
- {t("settings:providers.enablePromptCachingTitle")} -
-
- ) - } - return null - })()} ) } diff --git a/webview-ui/src/components/settings/providers/__tests__/Bedrock.spec.tsx b/webview-ui/src/components/settings/providers/__tests__/Bedrock.spec.tsx index b827024859..461a7ca61a 100644 --- a/webview-ui/src/components/settings/providers/__tests__/Bedrock.spec.tsx +++ b/webview-ui/src/components/settings/providers/__tests__/Bedrock.spec.tsx @@ -260,6 +260,22 @@ describe("Bedrock Component", () => { // Test Scenario 3: UI Elements Tests describe("UI Elements", () => { + it("does not render legacy provider-level prompt caching controls", () => { + const apiConfiguration: Partial = { + apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0", + awsUseProfile: true, + } + + render( + , + ) + + expect(screen.queryByText("settings:providers.cacheUsageNote")).not.toBeInTheDocument() + }) + it("should display example URLs when VPC endpoint checkbox is checked", () => { const apiConfiguration: Partial = { awsBedrockEndpoint: "https://example.com", diff --git a/webview-ui/src/context/ExtensionStateContext.tsx b/webview-ui/src/context/ExtensionStateContext.tsx index 85a750065f..6f900def14 100644 --- a/webview-ui/src/context/ExtensionStateContext.tsx +++ b/webview-ui/src/context/ExtensionStateContext.tsx @@ -124,8 +124,6 @@ export interface ExtensionStateContextType extends ExtensionState { setTelemetrySetting: (value: TelemetrySetting) => void remoteBrowserEnabled?: boolean setRemoteBrowserEnabled: (value: boolean) => void - awsUsePromptCache?: boolean - setAwsUsePromptCache: (value: boolean) => void maxImageFileSize: number setMaxImageFileSize: (value: number) => void maxTotalImageSize: number @@ -588,7 +586,6 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode setShowRooIgnoredFiles: (value) => setState((prevState) => ({ ...prevState, showRooIgnoredFiles: value })), setEnableSubfolderRules: (value) => setState((prevState) => ({ ...prevState, enableSubfolderRules: value })), setRemoteBrowserEnabled: (value) => setState((prevState) => ({ ...prevState, remoteBrowserEnabled: value })), - setAwsUsePromptCache: (value) => setState((prevState) => ({ ...prevState, awsUsePromptCache: value })), setMaxImageFileSize: (value) => setState((prevState) => ({ ...prevState, maxImageFileSize: value })), setMaxTotalImageSize: (value) => setState((prevState) => ({ ...prevState, maxTotalImageSize: value })), setPinnedApiConfigs: (value) => setState((prevState) => ({ ...prevState, pinnedApiConfigs: value })),