scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-08-28 05:27:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: unify prompt caching via AI SDK with legacy migration

Browse source
This commit is contained in:
Hannes Rudolph 2026-02-11 13:11:22 -07:00
parent d2c52c9e09
commit 4a3ebb52dd
24 changed files with 969 additions and 323 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
packages/types/src/provider-settings.ts
View file

@ -180,6 +180,9 @@ const baseProviderSettingsSchema = z.object({
modelTemperature: z.number().nullish(),
rateLimitSeconds: z.number().optional(),
consecutiveMistakeLimit: z.number().min(0).optional(),
promptCachingEnabled: z.boolean().optional(),
promptCachingStrategy: z.enum(["conservative", "balanced", "aggressive"]).optional(),
promptCachingProviderOverrides: z.record(z.string(), z.boolean()).optional(),
// Model reasoning.
enableReasoningEffort: z.boolean().optional(),
@ -217,7 +220,6 @@ const bedrockSchema = apiModelIdProviderModelSchema.extend({
awsRegion: z.string().optional(),
awsUseCrossRegionInference: z.boolean().optional(),
awsUseGlobalInference: z.boolean().optional(), // Enable Global Inference profile routing when supported
awsUsePromptCache: z.boolean().optional(),
awsProfile: z.string().optional(),
awsUseProfile: z.boolean().optional(),
awsApiKey: z.string().optional(),
@ -340,7 +342,6 @@ const litellmSchema = baseProviderSettingsSchema.extend({
litellmBaseUrl: z.string().optional(),
litellmApiKey: z.string().optional(),
litellmModelId: z.string().optional(),
litellmUsePromptCache: z.boolean().optional(),
})
const sambaNovaSchema = apiModelIdProviderModelSchema.extend({

21
src/api/providers/__tests__/anthropic.spec.ts
View file

@ -420,6 +420,27 @@ describe("AnthropicHandler", () => {
const systemMessages = callArgs.messages.filter((m: any) => m.role === "system")
expect(systemMessages).toHaveLength(0)
})
it("should disable prompt caching when globally disabled", async () => {
setupStreamTextMock([{ type: "text-delta", text: "test" }])
const cacheDisabledHandler = new AnthropicHandler({
...mockOptions,
promptCachingEnabled: false,
})
const stream = cacheDisabledHandler.createMessage(systemPrompt, [
{ role: "user", content: [{ type: "text" as const, text: "hello" }] },
])
for await (const _chunk of stream) {
// Consume
}
const callArgs = mockStreamText.mock.calls[0]![0]
expect(callArgs.systemProviderOptions).toBeUndefined()
expect(callArgs.messages[0].providerOptions).toBeUndefined()
})
})
describe("completePrompt", () => {

70
src/api/providers/__tests__/bedrock.spec.ts
View file

@ -591,6 +591,76 @@ describe("AwsBedrockHandler", () => {
})
})
describe("prompt caching policy", () => {
function setupMockStreamText() {
async function* mockFullStream() {
yield { type: "text-delta", text: "Response" }
}
mockStreamText.mockReturnValue({
fullStream: mockFullStream(),
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
providerMetadata: Promise.resolve({}),
})
}
it("disables cache markers when global prompt caching is off", async () => {
setupMockStreamText()
const cacheDisabledHandler = new AwsBedrockHandler({
apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
awsAccessKey: "test-access-key",
awsSecretKey: "test-secret-key",
awsRegion: "us-east-1",
promptCachingEnabled: false,
})
const generator = cacheDisabledHandler.createMessage("", [
{
role: "user",
content: "Test prompt",
},
])
for await (const _chunk of generator) {
// consume
}
const callArgs = mockStreamText.mock.calls[0][0]
expect(callArgs.systemProviderOptions).toBeUndefined()
expect(callArgs.messages[0].providerOptions).toBeUndefined()
})
it("allows provider override to re-enable cache markers", async () => {
setupMockStreamText()
const overrideEnabledHandler = new AwsBedrockHandler({
apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
awsAccessKey: "test-access-key",
awsSecretKey: "test-secret-key",
awsRegion: "us-east-1",
promptCachingEnabled: false,
promptCachingProviderOverrides: {
bedrock: true,
},
})
const generator = overrideEnabledHandler.createMessage("", [
{
role: "user",
content: "Test prompt",
},
])
for await (const _chunk of generator) {
// consume
}
const callArgs = mockStreamText.mock.calls[0][0]
expect(callArgs.systemProviderOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
expect(callArgs.messages[0].providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
})
})
describe("error handling and validation", () => {
it("should handle invalid regions gracefully", () => {
expect(() => {

13
src/api/providers/__tests__/minimax.spec.ts
View file

@ -366,6 +366,19 @@ describe("MiniMaxHandler", () => {
}).rejects.toThrow("MiniMax: API Error")
expect(mockHandleAiSdkError).toHaveBeenCalledWith(expect.any(Error), "MiniMax")
})
it("disables prompt caching when globally disabled", async () => {
mockStreamText.mockReturnValue(createMockStream([{ type: "text-delta", text: "OK" }]))
const handler = createHandler({
promptCachingEnabled: false,
})
await collectChunks(handler.createMessage(systemPrompt, messages))
const callArgs = mockStreamText.mock.calls[0]?.[0]
expect(callArgs.systemProviderOptions).toBeUndefined()
expect(callArgs.messages[0]?.providerOptions).toBeUndefined()
})
})
describe("completePrompt", () => {

57
src/api/providers/__tests__/openai-native.spec.ts
View file

@ -879,6 +879,63 @@ describe("OpenAiNativeHandler", () => {
const callArgs = mockStreamText.mock.calls[0][0]
expect(callArgs.providerOptions.openai.promptCacheRetention).toBeUndefined()
})
it("should not pass promptCacheRetention when globally disabled", async () => {
async function* mockFullStream() {
yield { type: "text-delta", text: "Response" }
}
mockStreamText.mockReturnValue({
fullStream: mockFullStream(),
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
providerMetadata: Promise.resolve({}),
content: Promise.resolve([]),
})
const h = new OpenAiNativeHandler({
...mockOptions,
apiModelId: "gpt-5.1",
promptCachingEnabled: false,
})
const stream = h.createMessage(systemPrompt, messages)
for await (const _ of stream) {
// consume
}
const callArgs = mockStreamText.mock.calls[0][0]
expect(callArgs.providerOptions.openai.promptCacheRetention).toBeUndefined()
})
it("should pass promptCacheRetention when provider override enables it", async () => {
async function* mockFullStream() {
yield { type: "text-delta", text: "Response" }
}
mockStreamText.mockReturnValue({
fullStream: mockFullStream(),
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5 }),
providerMetadata: Promise.resolve({}),
content: Promise.resolve([]),
})
const h = new OpenAiNativeHandler({
...mockOptions,
apiModelId: "gpt-5.1",
promptCachingEnabled: false,
promptCachingProviderOverrides: {
"openai-native": true,
},
})
const stream = h.createMessage(systemPrompt, messages)
for await (const _ of stream) {
// consume
}
const callArgs = mockStreamText.mock.calls[0][0]
expect(callArgs.providerOptions.openai.promptCacheRetention).toBe("24h")
})
})
describe("completePrompt", () => {

71
src/api/providers/anthropic-vertex.ts
View file

@ -19,13 +19,13 @@ import { shouldUseReasoningBudget } from "../../shared/api"
import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
import { getModelParams } from "../transform/model-params"
import {
convertToAiSdkMessages,
convertToolsForAiSdk,
processAiSdkStreamPart,
mapToolChoice,
handleAiSdkError,
yieldResponseMessage,
} from "../transform/ai-sdk"
import { applyPromptCacheToMessages } from "../transform/prompt-cache"
import { calculateApiCostAnthropic } from "../../shared/cost"
import { DEFAULT_HEADERS } from "./constants"
@ -119,45 +119,25 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
anthropicProviderOptions.disableParallelToolUse = true
}
/**
* Vertex API has specific limitations for prompt caching:
* 1. Maximum of 4 blocks can have cache_control
* 2. Only text blocks can be cached (images and other content types cannot)
* 3. Cache control can only be applied to user messages, not assistant messages
*
* Our caching strategy:
* - Cache the system prompt (1 block)
* - Cache the last text block of the second-to-last user message (1 block)
* - Cache the last text block of the last user message (1 block)
* This ensures we stay under the 4-block limit while maintaining effective caching
* for the most relevant context.
*/
const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } }
const userMsgIndices = messages.reduce(
(acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc),
[] as number[],
)
const targetIndices = new Set<number>()
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex)
if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex)
if (targetIndices.size > 0) {
this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption)
}
const promptCache = applyPromptCacheToMessages({
adapter: "anthropic",
overrideKey: "vertex",
messages: aiSdkMessages,
modelInfo: {
supportsPromptCache: modelConfig.info.supportsPromptCache,
promptCacheRetention: modelConfig.info.promptCacheRetention,
},
settings: this.options,
})
// Build streamText request
// Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values
const requestOptions: Parameters<typeof streamText>[0] = {
model: this.provider(modelConfig.id),
system: systemPrompt,
...({
systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
} as Record<string, unknown>),
...(promptCache.systemProviderOptions
? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record<string, unknown>)
: {}),
messages: aiSdkMessages,
temperature: modelConfig.temperature,
maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
@ -241,29 +221,6 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
}
}
/**
* Apply cacheControl providerOptions to the correct AI SDK messages by walking
* the original Anthropic messages and converted AI SDK messages in parallel.
*
* convertToAiSdkMessages() can split a single Anthropic user message (containing
* tool_results + text) into 2 AI SDK messages (tool role + user role). This method
* accounts for that split so cache control lands on the right message.
*/
private applyCacheControlToAiSdkMessages(
aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
targetIndices: Set<number>,
cacheProviderOption: Record<string, Record<string, unknown>>,
): void {
for (const idx of targetIndices) {
if (idx >= 0 && idx < aiSdkMessages.length) {
aiSdkMessages[idx].providerOptions = {
...aiSdkMessages[idx].providerOptions,
...cacheProviderOption,
}
}
}
}
getModel() {
const modelId = this.options.apiModelId
let id = modelId && modelId in vertexModels ? (modelId as VertexModelId) : vertexDefaultModelId

60
src/api/providers/anthropic.ts
View file

@ -17,13 +17,13 @@ import { shouldUseReasoningBudget } from "../../shared/api"
import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
import { getModelParams } from "../transform/model-params"
import {
convertToAiSdkMessages,
convertToolsForAiSdk,
processAiSdkStreamPart,
mapToolChoice,
handleAiSdkError,
yieldResponseMessage,
} from "../transform/ai-sdk"
import { applyPromptCacheToMessages } from "../transform/prompt-cache"
import { calculateApiCostAnthropic } from "../../shared/cost"
import { DEFAULT_HEADERS } from "./constants"
@ -105,34 +105,25 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
anthropicProviderOptions.disableParallelToolUse = true
}
// Apply cache control to user messages
// Strategy: cache the last 2 user messages (write-to-cache + read-from-cache)
const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } }
const userMsgIndices = messages.reduce(
(acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc),
[] as number[],
)
const targetIndices = new Set<number>()
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex)
if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex)
if (targetIndices.size > 0) {
this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption)
}
const promptCache = applyPromptCacheToMessages({
adapter: "anthropic",
overrideKey: "anthropic",
messages: aiSdkMessages,
modelInfo: {
supportsPromptCache: modelConfig.info.supportsPromptCache,
promptCacheRetention: modelConfig.info.promptCacheRetention,
},
settings: this.options,
})
// Build streamText request
// Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values
const requestOptions: Parameters<typeof streamText>[0] = {
model: this.provider(modelConfig.id),
system: systemPrompt,
...({
systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
} as Record<string, unknown>),
...(promptCache.systemProviderOptions
? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record<string, unknown>)
: {}),
messages: aiSdkMessages,
temperature: modelConfig.temperature,
maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
@ -216,29 +207,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
}
}
/**
* Apply cacheControl providerOptions to the correct AI SDK messages by walking
* the original Anthropic messages and converted AI SDK messages in parallel.
*
* convertToAiSdkMessages() can split a single Anthropic user message (containing
* tool_results + text) into 2 AI SDK messages (tool role + user role). This method
* accounts for that split so cache control lands on the right message.
*/
private applyCacheControlToAiSdkMessages(
aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
targetIndices: Set<number>,
cacheProviderOption: Record<string, Record<string, unknown>>,
): void {
for (const idx of targetIndices) {
if (idx >= 0 && idx < aiSdkMessages.length) {
aiSdkMessages[idx].providerOptions = {
...aiSdkMessages[idx].providerOptions,
...cacheProviderOption,
}
}
}
}
getModel() {
const modelId = this.options.apiModelId
let id = modelId && modelId in anthropicModels ? (modelId as AnthropicModelId) : anthropicDefaultModelId

116
src/api/providers/bedrock.ts
View file

@ -25,7 +25,6 @@ import { TelemetryService } from "@roo-code/telemetry"
import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
import {
convertToAiSdkMessages,
convertToolsForAiSdk,
processAiSdkStreamPart,
mapToolChoice,
@ -33,6 +32,7 @@ import {
yieldResponseMessage,
} from "../transform/ai-sdk"
import { getModelParams } from "../transform/model-params"
import { applyPromptCacheToMessages } from "../transform/prompt-cache"
import { shouldUseReasoningBudget } from "../../shared/api"
import { BaseProvider } from "./base-provider"
import { DEFAULT_HEADERS } from "./constants"
@ -251,76 +251,25 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
}
}
// Prompt caching: use AI SDK's cachePoint mechanism
// The AI SDK's @ai-sdk/amazon-bedrock supports cachePoint in providerOptions per message.
//
// Strategy: Bedrock allows up to 4 cache checkpoints. We use them as:
// 1. System prompt (via systemProviderOptions below)
// 2-4. Up to 3 user messages in the conversation history
//
// For the message cache points, we target the last 2 user messages (matching
// Anthropic's strategy: write-to-cache + read-from-cache) PLUS an earlier "anchor"
// user message near the middle of the conversation. This anchor ensures the 20-block
// lookback window has a stable cache entry to hit, covering all assistant/tool messages
// between the anchor and the recent messages.
//
// We identify targets in the ORIGINAL Anthropic messages (before AI SDK conversion)
// because convertToAiSdkMessages() splits user messages containing tool_results into
// separate "tool" + "user" role messages, which would skew naive counting.
const usePromptCache = Boolean(this.options.awsUsePromptCache && this.supportsAwsPromptCache(modelConfig))
if (usePromptCache) {
const cachePointOption = { bedrock: { cachePoint: { type: "default" as const } } }
// Find all user message indices in the original (pre-conversion) message array.
const originalUserIndices = filteredMessages.reduce<number[]>(
(acc, msg, idx) => ("role" in msg && msg.role === "user" ? [...acc, idx] : acc),
[],
)
// Select up to 3 user messages for cache points (system prompt uses the 4th):
// - Last user message: write to cache for next request
// - Second-to-last user message: read from cache for current request
// - An "anchor" message earlier in the conversation for 20-block window coverage
const targetOriginalIndices = new Set<number>()
const numUserMsgs = originalUserIndices.length
if (numUserMsgs >= 1) {
// Always cache the last user message
targetOriginalIndices.add(originalUserIndices[numUserMsgs - 1])
}
if (numUserMsgs >= 2) {
// Cache the second-to-last user message
targetOriginalIndices.add(originalUserIndices[numUserMsgs - 2])
}
if (numUserMsgs >= 5) {
// Add an anchor cache point roughly in the first third of user messages.
// This ensures that the 20-block lookback from the second-to-last breakpoint
// can find a stable cache entry, covering all the assistant and tool messages
// in the middle of the conversation. We pick the user message at ~1/3 position.
const anchorIdx = Math.floor(numUserMsgs / 3)
// Only add if it's not already one of the last-2 targets
if (!targetOriginalIndices.has(originalUserIndices[anchorIdx])) {
targetOriginalIndices.add(originalUserIndices[anchorIdx])
}
}
// Apply cachePoint to the correct AI SDK messages by walking both arrays in parallel.
// A single original user message with tool_results becomes [tool-role msg, user-role msg]
// in the AI SDK array, while a plain user message becomes [user-role msg].
if (targetOriginalIndices.size > 0) {
this.applyCachePointsToAiSdkMessages(aiSdkMessages, targetOriginalIndices, cachePointOption)
}
}
const promptCache = applyPromptCacheToMessages({
adapter: "bedrock",
overrideKey: "bedrock",
messages: aiSdkMessages,
modelInfo: {
supportsPromptCache: modelConfig.info.supportsPromptCache,
promptCacheRetention: modelConfig.info.promptCacheRetention,
},
settings: this.options,
})
// Build streamText request
// Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values
const requestOptions: Parameters<typeof streamText>[0] = {
model: this.provider(modelConfig.id),
system: systemPrompt,
...(usePromptCache && {
systemProviderOptions: { bedrock: { cachePoint: { type: "default" } } } as Record<string, unknown>,
}),
...(promptCache.systemProviderOptions
? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record<string, unknown>)
: {}),
messages: aiSdkMessages,
temperature: modelConfig.temperature ?? (this.options.modelTemperature as number),
maxOutputTokens: modelConfig.maxTokens || (modelConfig.info.maxTokens as number),
@ -692,43 +641,6 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
}
}
/************************************************************************************
*
* CACHE
*
*************************************************************************************/
private supportsAwsPromptCache(modelConfig: { id: BedrockModelId | string; info: ModelInfo }): boolean | undefined {
return (
modelConfig?.info?.supportsPromptCache &&
(modelConfig?.info as any)?.cachableFields &&
(modelConfig?.info as any)?.cachableFields?.length > 0
)
}
/**
* Apply cachePoint providerOptions to the correct AI SDK messages by walking
* the original Anthropic messages and converted AI SDK messages in parallel.
*
* convertToAiSdkMessages() can split a single Anthropic user message (containing
* tool_results + text) into 2 AI SDK messages (tool role + user role). This method
* accounts for that split so cache points land on the right message.
*/
private applyCachePointsToAiSdkMessages(
aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
targetIndices: Set<number>,
cachePointOption: Record<string, Record<string, unknown>>,
): void {
for (const idx of targetIndices) {
if (idx >= 0 && idx < aiSdkMessages.length) {
aiSdkMessages[idx].providerOptions = {
...aiSdkMessages[idx].providerOptions,
...cachePointOption,
}
}
}
}
/************************************************************************************
*
* AMAZON REGIONS

51
src/api/providers/minimax.ts
View file

@ -9,13 +9,13 @@ import type { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
import { getModelParams } from "../transform/model-params"
import { mergeEnvironmentDetailsForMiniMax } from "../transform/minimax-format"
import {
convertToAiSdkMessages,
convertToolsForAiSdk,
processAiSdkStreamPart,
mapToolChoice,
handleAiSdkError,
yieldResponseMessage,
} from "../transform/ai-sdk"
import { applyPromptCacheToMessages } from "../transform/prompt-cache"
import { calculateApiCostAnthropic } from "../../shared/cost"
import { DEFAULT_HEADERS } from "./constants"
@ -72,7 +72,7 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand
})
const mergedMessages = mergeEnvironmentDetailsForMiniMax(messages as any)
const aiSdkMessages = mergedMessages as ModelMessage[]
const aiSdkMessages = (mergedMessages as ModelMessage[]).map((message) => ({ ...message }))
const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
@ -89,29 +89,23 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand
anthropicProviderOptions.disableParallelToolUse = true
}
const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } }
const userMsgIndices = mergedMessages.reduce(
(acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc),
[] as number[],
)
const targetIndices = new Set<number>()
const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex)
if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex)
if (targetIndices.size > 0) {
this.applyCacheControlToAiSdkMessages(aiSdkMessages, targetIndices, cacheProviderOption)
}
const promptCache = applyPromptCacheToMessages({
adapter: "anthropic",
overrideKey: "minimax",
messages: aiSdkMessages,
modelInfo: {
supportsPromptCache: modelConfig.info.supportsPromptCache,
promptCacheRetention: (modelConfig.info as ModelInfo).promptCacheRetention,
},
settings: this.options,
})
const requestOptions = {
model: this.client(modelConfig.id),
system: systemPrompt,
...({
systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
} as Record<string, unknown>),
...(promptCache.systemProviderOptions
? ({ systemProviderOptions: promptCache.systemProviderOptions } as Record<string, unknown>)
: {}),
messages: aiSdkMessages,
temperature: modelParams.temperature,
maxOutputTokens: modelParams.maxTokens ?? modelConfig.info.maxTokens,
@ -187,21 +181,6 @@ export class MiniMaxHandler extends BaseProvider implements SingleCompletionHand
}
}
private applyCacheControlToAiSdkMessages(
aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
targetIndices: Set<number>,
cacheProviderOption: Record<string, Record<string, unknown>>,
): void {
for (const idx of targetIndices) {
if (idx >= 0 && idx < aiSdkMessages.length) {
aiSdkMessages[idx].providerOptions = {
...aiSdkMessages[idx].providerOptions,
...cacheProviderOption,
}
}
}
}
getModel() {
const modelId = this.options.apiModelId

33
src/api/providers/openai-native.ts
View file

@ -19,13 +19,8 @@ import {
import type { ApiHandlerOptions } from "../../shared/api"
import { calculateApiCostOpenAI } from "../../shared/cost"
import {
convertToAiSdkMessages,
convertToolsForAiSdk,
consumeAiSdkStream,
mapToolChoice,
handleAiSdkError,
} from "../transform/ai-sdk"
import { convertToolsForAiSdk, consumeAiSdkStream, mapToolChoice, handleAiSdkError } from "../transform/ai-sdk"
import { applyPromptCacheToMessages } from "../transform/prompt-cache"
import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
import { getModelParams } from "../transform/model-params"
@ -265,15 +260,6 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
return selected && selected !== "disable" ? (selected as any) : undefined
}
/**
* Returns the appropriate prompt cache retention policy for the given model, if any.
*/
private getPromptCacheRetention(model: OpenAiNativeModel): "24h" | undefined {
if (!model.info.supportsPromptCache) return undefined
if (model.info.promptCacheRetention === "24h") return "24h"
return undefined
}
/**
* Returns a shallow-cloned ModelInfo with pricing overridden for the given tier, if available.
*/
@ -301,7 +287,16 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
systemPrompt?: string,
): Record<string, any> {
const reasoningEffort = this.getReasoningEffort(model)
const promptCacheRetention = this.getPromptCacheRetention(model)
const promptCache = applyPromptCacheToMessages({
adapter: "openai-native",
overrideKey: "openai-native",
messages: [],
modelInfo: {
supportsPromptCache: model.info.supportsPromptCache,
promptCacheRetention: model.info.promptCacheRetention,
},
settings: this.options,
})
const requestedTier = (this.options.openAiNativeServiceTier as ServiceTier | undefined) || undefined
const allowedTierNames = new Set(model.info.tiers?.map((t) => t.name).filter(Boolean) || [])
@ -329,8 +324,8 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
openaiOptions.serviceTier = requestedTier
}
if (promptCacheRetention) {
openaiOptions.promptCacheRetention = promptCacheRetention
if (promptCache.providerOptionsPatch?.openai?.promptCacheRetention) {
openaiOptions.promptCacheRetention = promptCache.providerOptionsPatch.openai.promptCacheRetention
}
return { openai: openaiOptions }

210
src/api/transform/__tests__/prompt-cache.spec.ts Normal file
View file

@ -0,0 +1,210 @@
import type { ModelMessage } from "ai"
import { applyPromptCacheToMessages, resolvePromptCachePolicy } from "../prompt-cache"
describe("prompt-cache", () => {
describe("resolvePromptCachePolicy", () => {
it("defaults to enabled with aggressive strategy", () => {
const policy = resolvePromptCachePolicy({
overrideKey: "bedrock",
supportsPromptCache: true,
})
expect(policy).toEqual({
enabled: true,
strategy: "aggressive",
})
})
it("uses provider override over global setting", () => {
const disabledByGlobal = resolvePromptCachePolicy({
overrideKey: "bedrock",
supportsPromptCache: true,
settings: {
promptCachingEnabled: false,
},
})
expect(disabledByGlobal.enabled).toBe(false)
const enabledByOverride = resolvePromptCachePolicy({
overrideKey: "bedrock",
supportsPromptCache: true,
settings: {
promptCachingEnabled: false,
promptCachingProviderOverrides: {
bedrock: true,
},
},
})
expect(enabledByOverride.enabled).toBe(true)
const disabledByOverride = resolvePromptCachePolicy({
overrideKey: "bedrock",
supportsPromptCache: true,
settings: {
promptCachingEnabled: true,
promptCachingProviderOverrides: {
bedrock: false,
},
},
})
expect(disabledByOverride.enabled).toBe(false)
})
it("disables caching for unsupported models", () => {
const policy = resolvePromptCachePolicy({
overrideKey: "anthropic",
supportsPromptCache: false,
settings: {
promptCachingEnabled: true,
},
})
expect(policy.enabled).toBe(false)
})
})
describe("applyPromptCacheToMessages", () => {
function buildMessages(): ModelMessage[] {
return [
{
role: "user",
content: [{ type: "text", text: "u1" }],
},
{
role: "assistant",
content: [{ type: "text", text: "a1" }],
},
{
role: "user",
content: [{ type: "text", text: "u2" }],
},
{
role: "assistant",
content: [{ type: "text", text: "a2" }],
},
{
role: "user",
content: [{ type: "text", text: "u3" }],
},
]
}
it("applies anthropic strategy and system marker", () => {
const messages = buildMessages()
const result = applyPromptCacheToMessages({
adapter: "anthropic",
overrideKey: "anthropic",
messages,
modelInfo: {
supportsPromptCache: true,
},
settings: {
promptCachingStrategy: "aggressive",
},
})
expect(result.systemProviderOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
})
expect((messages[0] as any).providerOptions).toBeUndefined()
expect((messages[2] as any).providerOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
})
expect((messages[4] as any).providerOptions).toEqual({
anthropic: { cacheControl: { type: "ephemeral" } },
})
})
it("applies bedrock aggressive checkpoints to last three user messages", () => {
const messages = buildMessages()
const result = applyPromptCacheToMessages({
adapter: "bedrock",
overrideKey: "bedrock",
messages,
modelInfo: {
supportsPromptCache: true,
},
settings: {
promptCachingStrategy: "aggressive",
},
})
expect(result.systemProviderOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
expect((messages[0] as any).providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
expect((messages[2] as any).providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
expect((messages[4] as any).providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
})
it("applies balanced strategy with fewer checkpoints", () => {
const messages = buildMessages()
applyPromptCacheToMessages({
adapter: "bedrock",
overrideKey: "bedrock",
messages,
modelInfo: {
supportsPromptCache: true,
},
settings: {
promptCachingStrategy: "balanced",
},
})
expect((messages[0] as any).providerOptions).toBeUndefined()
expect((messages[2] as any).providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
expect((messages[4] as any).providerOptions).toEqual({
bedrock: { cachePoint: { type: "default" } },
})
})
it("returns openai retention patch when enabled", () => {
const messages: ModelMessage[] = []
const result = applyPromptCacheToMessages({
adapter: "openai-native",
overrideKey: "openai-native",
messages,
modelInfo: {
supportsPromptCache: true,
promptCacheRetention: "24h",
},
})
expect(result.providerOptionsPatch).toEqual({
openai: {
promptCacheRetention: "24h",
},
})
})
it("does not return openai retention patch when globally disabled", () => {
const result = applyPromptCacheToMessages({
adapter: "openai-native",
overrideKey: "openai-native",
messages: [],
modelInfo: {
supportsPromptCache: true,
promptCacheRetention: "24h",
},
settings: {
promptCachingEnabled: false,
},
})
expect(result.enabled).toBe(false)
expect(result.providerOptionsPatch).toBeUndefined()
})
})
})

175
src/api/transform/prompt-cache.ts Normal file
View file

@ -0,0 +1,175 @@
import type { ModelInfo, ProviderSettings } from "@roo-code/types"
import type { ModelMessage } from "ai"
export type PromptCachingStrategy = NonNullable<ProviderSettings["promptCachingStrategy"]>
export type PromptCacheAdapter = "anthropic" | "bedrock" | "openai-native"
export interface PromptCachePolicy {
enabled: boolean
strategy: PromptCachingStrategy
}
export interface ApplyPromptCacheArgs {
adapter: PromptCacheAdapter
overrideKey: string
messages: ModelMessage[]
modelInfo: Pick<ModelInfo, "supportsPromptCache" | "promptCacheRetention">
settings?: Pick<
ProviderSettings,
"promptCachingEnabled" | "promptCachingStrategy" | "promptCachingProviderOverrides"
>
}
export interface AppliedPromptCache {
enabled: boolean
strategy: PromptCachingStrategy
systemProviderOptions?: Record<string, unknown>
providerOptionsPatch?: Record<string, Record<string, unknown>>
}
const DEFAULT_PROMPT_CACHING_STRATEGY: PromptCachingStrategy = "aggressive"
export function resolvePromptCachePolicy({
overrideKey,
settings,
supportsPromptCache,
}: {
overrideKey: string
settings?: Pick<
ProviderSettings,
"promptCachingEnabled" | "promptCachingStrategy" | "promptCachingProviderOverrides"
>
supportsPromptCache: boolean
}): PromptCachePolicy {
const strategy = settings?.promptCachingStrategy ?? DEFAULT_PROMPT_CACHING_STRATEGY
if (!supportsPromptCache) {
return { enabled: false, strategy }
}
const globalEnabled = settings?.promptCachingEnabled ?? true
const providerOverride = settings?.promptCachingProviderOverrides?.[overrideKey]
const enabled = providerOverride ?? globalEnabled
return { enabled, strategy }
}
export function applyPromptCacheToMessages({
adapter,
overrideKey,
messages,
modelInfo,
settings,
}: ApplyPromptCacheArgs): AppliedPromptCache {
const policy = resolvePromptCachePolicy({
overrideKey,
settings,
supportsPromptCache: modelInfo.supportsPromptCache,
})
if (!policy.enabled) {
return {
enabled: false,
strategy: policy.strategy,
}
}
if (adapter === "openai-native") {
if (modelInfo.promptCacheRetention === "24h") {
return {
enabled: true,
strategy: policy.strategy,
providerOptionsPatch: {
openai: {
promptCacheRetention: "24h",
},
},
}
}
return {
enabled: true,
strategy: policy.strategy,
}
}
const adapterConfig = getMessageAdapterConfig(adapter)
const checkpointCount = resolveCheckpointCount(policy.strategy, adapterConfig.maxUserCheckpoints)
const userIndices = getUserMessageIndices(messages)
const targetIndices = userIndices.slice(-checkpointCount)
applyProviderOptionAtIndices(messages, targetIndices, adapterConfig.messageProviderOption)
return {
enabled: true,
strategy: policy.strategy,
systemProviderOptions: adapterConfig.systemProviderOptions,
}
}
function getMessageAdapterConfig(adapter: Exclude<PromptCacheAdapter, "openai-native">): {
maxUserCheckpoints: number
systemProviderOptions: Record<string, unknown>
messageProviderOption: Record<string, Record<string, unknown>>
} {
if (adapter === "bedrock") {
return {
maxUserCheckpoints: 3,
systemProviderOptions: {
bedrock: { cachePoint: { type: "default" } },
},
messageProviderOption: {
bedrock: { cachePoint: { type: "default" } },
},
}
}
return {
maxUserCheckpoints: 2,
systemProviderOptions: {
anthropic: { cacheControl: { type: "ephemeral" } },
},
messageProviderOption: {
anthropic: { cacheControl: { type: "ephemeral" } },
},
}
}
function resolveCheckpointCount(strategy: PromptCachingStrategy, maxUserCheckpoints: number): number {
if (maxUserCheckpoints <= 0) {
return 0
}
if (strategy === "conservative") {
return 1
}
if (strategy === "balanced") {
return Math.max(1, Math.ceil(maxUserCheckpoints / 2))
}
return maxUserCheckpoints
}
function getUserMessageIndices(messages: ModelMessage[]): number[] {
const indices: number[] = []
for (let i = 0; i < messages.length; i++) {
if (messages[i].role === "user") {
indices.push(i)
}
}
return indices
}
function applyProviderOptionAtIndices(
messages: ModelMessage[],
indices: number[],
providerOption: Record<string, Record<string, unknown>>,
): void {
for (const index of indices) {
const message = messages[index] as ModelMessage & { providerOptions?: unknown }
message.providerOptions = {
...((message.providerOptions as Record<string, unknown> | undefined) ?? {}),
...providerOption,
} as any
}
}

50
src/core/config/ContextProxy.ts
View file

@ -22,6 +22,7 @@ import { TelemetryService } from "@roo-code/telemetry"
import { logger } from "../../utils/logging"
import { supportPrompt } from "../../shared/support-prompt"
import { migrateLegacyPromptCacheSettings } from "./migrateLegacyPromptCacheSettings"
type GlobalStateKey = keyof GlobalState
type SecretStateKey = keyof SecretState
@ -94,6 +95,9 @@ export class ContextProxy {
// Migration: Sanitize invalid/removed API providers
await this.migrateInvalidApiProvider()
// Migration: one-time read of removed provider-level prompt cache keys.
await this.migrateLegacyPromptCacheKeys()
// Migration: Move legacy customCondensingPrompt to customSupportPrompts
await this.migrateLegacyCondensingPrompt()
@ -103,6 +107,45 @@ export class ContextProxy {
this._isInitialized = true
}
/**
* Migrates removed provider-level prompt cache toggles to the unified
* `promptCachingProviderOverrides` map and clears the legacy keys.
*/
private async migrateLegacyPromptCacheKeys() {
try {
const rawAwsToggle = this.originalContext.globalState.get<unknown>("awsUsePromptCache" as any)
const rawLiteLlmToggle = this.originalContext.globalState.get<unknown>("litellmUsePromptCache" as any)
if (rawAwsToggle === undefined && rawLiteLlmToggle === undefined) {
return
}
const migrationInput: Record<string, unknown> = {
...this.stateCache,
awsUsePromptCache: rawAwsToggle,
litellmUsePromptCache: rawLiteLlmToggle,
}
const migration = migrateLegacyPromptCacheSettings(migrationInput)
if (!migration.changed) {
return
}
const overrides = migration.config.promptCachingProviderOverrides as Record<string, boolean> | undefined
await this.originalContext.globalState.update("promptCachingProviderOverrides", overrides)
this.stateCache.promptCachingProviderOverrides = overrides
await this.originalContext.globalState.update("awsUsePromptCache" as any, undefined)
await this.originalContext.globalState.update("litellmUsePromptCache" as any, undefined)
delete (this.stateCache as Record<string, unknown>).awsUsePromptCache
delete (this.stateCache as Record<string, unknown>).litellmUsePromptCache
} catch (error) {
logger.error(
`Error during legacy prompt cache migration: ${error instanceof Error ? error.message : String(error)}`,
)
}
}
/**
* Migrates the legacy customCondensingPrompt to the new customSupportPrompts structure
* and removes the legacy field.
@ -459,6 +502,13 @@ export class ContextProxy {
}
}
const promptCacheMigration = migrateLegacyPromptCacheSettings({
...(sanitizedValues as unknown as Record<string, unknown>),
awsUsePromptCache: (this.stateCache as Record<string, unknown>).awsUsePromptCache,
litellmUsePromptCache: (this.stateCache as Record<string, unknown>).litellmUsePromptCache,
})
sanitizedValues = promptCacheMigration.config as RooCodeSettings
const isKnownProvider =
typeof values.apiProvider === "string" &&
(isProviderName(values.apiProvider) || isRetiredProvider(values.apiProvider))

26
src/core/config/ProviderSettingsManager.ts
View file

@ -18,6 +18,7 @@ import { TelemetryService } from "@roo-code/telemetry"
import { Mode, modes } from "../../shared/modes"
import { buildApiHandler } from "../../api"
import { migrateLegacyPromptCacheSettings } from "./migrateLegacyPromptCacheSettings"
// Type-safe model migrations mapping
type ModelMigrations = {
@ -128,6 +129,10 @@ export class ProviderSettingsManager {
isDirty = true
}
if (this.applyLegacyPromptCacheMigration(providerProfiles)) {
isDirty = true
}
// Ensure all configs have IDs.
for (const [_name, apiConfig] of Object.entries(providerProfiles.apiConfigs)) {
if (!apiConfig.id) {
@ -313,6 +318,22 @@ export class ProviderSettingsManager {
return migrated
}
private applyLegacyPromptCacheMigration(providerProfiles: ProviderProfiles): boolean {
let migrated = false
for (const [name, apiConfig] of Object.entries(providerProfiles.apiConfigs)) {
const migrationResult = migrateLegacyPromptCacheSettings(apiConfig as unknown as Record<string, unknown>)
if (!migrationResult.changed) {
continue
}
providerProfiles.apiConfigs[name] = migrationResult.config as ProviderSettingsWithId
migrated = true
}
return migrated
}
/**
* Clean model ID by removing prefix before "/"
*/
@ -645,7 +666,8 @@ export class ProviderSettingsManager {
return apiConfig
}
const config = apiConfig as Record<string, unknown>
const migrationResult = migrateLegacyPromptCacheSettings(apiConfig as Record<string, unknown>)
const config = migrationResult.config as Record<string, unknown>
const apiProvider = config.apiProvider
@ -663,7 +685,7 @@ export class ProviderSettingsManager {
return restConfig
}
return apiConfig
return config
}
private async store(providerProfiles: ProviderProfiles) {

33
src/core/config/__tests__/ContextProxy.spec.ts
View file

@ -70,16 +70,20 @@ describe("ContextProxy", () => {
describe("constructor", () => {
it("should initialize state cache with all global state keys", () => {
// +3 for the migration checks:
// +5 for the migration checks:
// 1. openRouterImageGenerationSettings
// 2. customCondensingPrompt
// 3. customSupportPrompts (for migrateOldDefaultCondensingPrompt)
expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 3)
// 2. awsUsePromptCache
// 3. litellmUsePromptCache
// 4. customCondensingPrompt
// 5. customSupportPrompts (for migrateOldDefaultCondensingPrompt)
expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 5)
for (const key of GLOBAL_STATE_KEYS) {
expect(mockGlobalState.get).toHaveBeenCalledWith(key)
}
// Also check for migration calls
expect(mockGlobalState.get).toHaveBeenCalledWith("openRouterImageGenerationSettings")
expect(mockGlobalState.get).toHaveBeenCalledWith("awsUsePromptCache")
expect(mockGlobalState.get).toHaveBeenCalledWith("litellmUsePromptCache")
expect(mockGlobalState.get).toHaveBeenCalledWith("customCondensingPrompt")
expect(mockGlobalState.get).toHaveBeenCalledWith("customSupportPrompts")
})
@ -104,8 +108,8 @@ describe("ContextProxy", () => {
const result = proxy.getGlobalState("apiProvider")
expect(result).toBe("deepseek")
// Original context should be called once during updateGlobalState (+3 for migration checks)
expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 3) // From initialization + migration checks
// Original context should be called once during updateGlobalState (+5 for migration checks)
expect(mockGlobalState.get).toHaveBeenCalledTimes(GLOBAL_STATE_KEYS.length + 5) // From initialization + migration checks
})
it("should handle default values correctly", async () => {
@ -553,6 +557,23 @@ describe("ContextProxy", () => {
// Should not throw and should return undefined
expect(settings.apiProvider).toBeUndefined()
})
it("should migrate legacy prompt cache toggles to provider overrides", async () => {
await proxy.setValues({
apiProvider: "bedrock",
awsUsePromptCache: false as any,
litellmUsePromptCache: false as any,
} as any)
const settings = proxy.getProviderSettings()
expect((settings as any).awsUsePromptCache).toBeUndefined()
expect((settings as any).litellmUsePromptCache).toBeUndefined()
expect(settings.promptCachingProviderOverrides).toEqual({
bedrock: false,
litellm: false,
})
})
})
describe("old default condensing prompt migration", () => {

39
src/core/config/__tests__/ProviderSettingsManager.spec.ts
View file

@ -336,6 +336,45 @@ describe("ProviderSettingsManager", () => {
expect(storedConfig.apiConfigs.default.apiModelId).toEqual("roo/code-supernova-1-million")
})
it("should migrate legacy prompt cache toggles to provider overrides", async () => {
mockSecrets.get.mockResolvedValue(
JSON.stringify({
currentApiConfigName: "default",
apiConfigs: {
default: {
id: "default",
apiProvider: "bedrock",
awsUsePromptCache: false,
},
lite: {
id: "lite",
apiProvider: "litellm",
litellmUsePromptCache: false,
},
},
migrations: {
rateLimitSecondsMigrated: true,
openAiHeadersMigrated: true,
consecutiveMistakeLimitMigrated: true,
todoListEnabledMigrated: true,
claudeCodeLegacySettingsMigrated: true,
},
}),
)
await providerSettingsManager.initialize()
expect(mockSecrets.store).toHaveBeenCalled()
const calls = mockSecrets.store.mock.calls
const storedConfig = JSON.parse(calls[calls.length - 1][1])
expect(storedConfig.apiConfigs.default.awsUsePromptCache).toBeUndefined()
expect(storedConfig.apiConfigs.default.promptCachingProviderOverrides).toEqual({ bedrock: false })
expect(storedConfig.apiConfigs.lite.litellmUsePromptCache).toBeUndefined()
expect(storedConfig.apiConfigs.lite.promptCachingProviderOverrides).toEqual({ litellm: false })
})
it("should throw error if secrets storage fails", async () => {
mockSecrets.get.mockRejectedValue(new Error("Storage failed"))

67
src/core/config/__tests__/migrateLegacyPromptCacheSettings.spec.ts Normal file
View file

@ -0,0 +1,67 @@
import { migrateLegacyPromptCacheSettings } from "../migrateLegacyPromptCacheSettings"
describe("migrateLegacyPromptCacheSettings", () => {
it("maps legacy false toggles to provider overrides and removes legacy keys", () => {
const input = {
apiProvider: "bedrock",
awsUsePromptCache: false,
litellmUsePromptCache: false,
}
const result = migrateLegacyPromptCacheSettings(input)
expect(result.changed).toBe(true)
expect(result.config).toEqual({
apiProvider: "bedrock",
promptCachingProviderOverrides: {
bedrock: false,
litellm: false,
},
})
})
it("does not create overrides for legacy true toggles and still removes legacy keys", () => {
const input = {
apiProvider: "bedrock",
awsUsePromptCache: true,
litellmUsePromptCache: true,
}
const result = migrateLegacyPromptCacheSettings(input)
expect(result.changed).toBe(true)
expect(result.config).toEqual({
apiProvider: "bedrock",
})
})
it("does not overwrite explicit new-format overrides", () => {
const input = {
awsUsePromptCache: false,
promptCachingProviderOverrides: {
bedrock: true,
},
}
const result = migrateLegacyPromptCacheSettings(input)
expect(result.changed).toBe(true)
expect(result.config).toEqual({
promptCachingProviderOverrides: {
bedrock: true,
},
})
})
it("returns unchanged when no legacy keys exist", () => {
const input = {
apiProvider: "anthropic",
promptCachingEnabled: true,
}
const result = migrateLegacyPromptCacheSettings(input)
expect(result.changed).toBe(false)
expect(result.config).toEqual(input)
})
})

50
src/core/config/migrateLegacyPromptCacheSettings.ts Normal file
View file

@ -0,0 +1,50 @@
type PromptCacheMigrationInput = Record<string, unknown>
export interface PromptCacheMigrationResult<T extends PromptCacheMigrationInput> {
config: T
changed: boolean
}
/**
* One-time migration helper for legacy provider-specific prompt cache toggles.
* - Maps legacy `false` values to provider overrides
* - Drops legacy keys from the config object
*/
export function migrateLegacyPromptCacheSettings<T extends PromptCacheMigrationInput>(
config: T,
): PromptCacheMigrationResult<T> {
let changed = false
const next = { ...config } as Record<string, unknown>
const currentOverrides = next.promptCachingProviderOverrides
const overrides =
typeof currentOverrides === "object" && currentOverrides !== null && !Array.isArray(currentOverrides)
? { ...(currentOverrides as Record<string, unknown>) }
: {}
if (next.awsUsePromptCache === false && overrides.bedrock === undefined) {
overrides.bedrock = false
changed = true
}
if (next.litellmUsePromptCache === false && overrides.litellm === undefined) {
overrides.litellm = false
changed = true
}
if (Object.keys(overrides).length > 0) {
next.promptCachingProviderOverrides = overrides
}
if ("awsUsePromptCache" in next) {
delete next.awsUsePromptCache
changed = true
}
if ("litellmUsePromptCache" in next) {
delete next.litellmUsePromptCache
changed = true
}
return { config: next as T, changed }
}

36
webview-ui/src/components/settings/ApiOptions.tsx
View file

@ -3,6 +3,7 @@ import { convertHeadersToObject } from "./utils/headers"
import { useDebounce } from "react-use"
import { VSCodeLink } from "@vscode/webview-ui-toolkit/react"
import { ExternalLinkIcon } from "@radix-ui/react-icons"
import { Checkbox } from "vscrui"
import {
type ProviderName,
@ -573,7 +574,6 @@ const ApiOptions = ({
<Bedrock
apiConfiguration={apiConfiguration}
setApiConfigurationField={setApiConfigurationField}
selectedModelInfo={selectedModelInfo}
simplifySettings={fromWelcomeView}
/>
)}
@ -794,6 +794,40 @@ const ApiOptions = ({
}
onChange={(value) => setApiConfigurationField("consecutiveMistakeLimit", value)}
/>
<Checkbox
checked={apiConfiguration.promptCachingEnabled ?? true}
onChange={(checked: boolean) =>
setApiConfigurationField("promptCachingEnabled", checked)
}>
<div className="flex items-center gap-1">
<span>{t("settings:providers.enablePromptCaching")}</span>
</div>
</Checkbox>
<div className="text-sm text-vscode-descriptionForeground ml-6 -mt-2">
{t("settings:providers.enablePromptCachingTitle")}
</div>
{(apiConfiguration.promptCachingEnabled ?? true) && (
<div>
<label className="block font-medium mb-1">Prompt caching strategy</label>
<Select
value={apiConfiguration.promptCachingStrategy || "aggressive"}
onValueChange={(value) =>
setApiConfigurationField(
"promptCachingStrategy",
value as ProviderSettings["promptCachingStrategy"],
)
}>
<SelectTrigger className="w-full">
<SelectValue placeholder={t("settings:common.select")} />
</SelectTrigger>
<SelectContent>
<SelectItem value="conservative">Conservative</SelectItem>
<SelectItem value="balanced">Balanced</SelectItem>
<SelectItem value="aggressive">Aggressive</SelectItem>
</SelectContent>
</Select>
</div>
)}
{selectedProvider === "openrouter" &&
openRouterModelProviders &&
Object.keys(openRouterModelProviders).length > 0 && (

37
webview-ui/src/components/settings/__tests__/ApiOptions.spec.tsx
View file

@ -304,6 +304,43 @@ describe("ApiOptions", () => {
expect(screen.getByTestId("rate-limit-seconds-control")).toBeInTheDocument()
})
it("shows global prompt caching controls in advanced settings", () => {
renderApiOptions({
apiConfiguration: {},
})
expect(screen.getByText("settings:providers.enablePromptCaching")).toBeInTheDocument()
expect(screen.getByText("Prompt caching strategy")).toBeInTheDocument()
})
it("updates prompt caching fields from advanced settings controls", () => {
const mockSetApiConfigurationField = vi.fn()
renderApiOptions({
apiConfiguration: {},
setApiConfigurationField: mockSetApiConfigurationField,
})
const enablePromptCachingLabel = screen.getByText("settings:providers.enablePromptCaching").closest("label")
const enablePromptCachingInput = enablePromptCachingLabel?.querySelector("input") as HTMLInputElement
fireEvent.click(enablePromptCachingInput)
expect(mockSetApiConfigurationField).toHaveBeenCalledWith("promptCachingEnabled", false)
const conservativeOption = screen.getByText("Conservative")
const strategySelect = conservativeOption.closest("select") as HTMLSelectElement
fireEvent.change(strategySelect, { target: { value: "conservative" } })
expect(mockSetApiConfigurationField).toHaveBeenCalledWith("promptCachingStrategy", expect.any(String))
})
it("hides prompt caching strategy selector when prompt caching is disabled", () => {
renderApiOptions({
apiConfiguration: {
promptCachingEnabled: false,
},
})
expect(screen.queryByText("Prompt caching strategy")).not.toBeInTheDocument()
})
it("hides all controls when fromWelcomeView is true", () => {
renderApiOptions({ fromWelcomeView: true })
expect(screen.queryByTestId("temperature-control")).not.toBeInTheDocument()

28
webview-ui/src/components/settings/providers/Bedrock.tsx
View file

@ -4,7 +4,6 @@ import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react"
import {
type ProviderSettings,
type ModelInfo,
type BedrockServiceTier,
BEDROCK_REGIONS,
BEDROCK_1M_CONTEXT_MODEL_IDS,
@ -13,18 +12,17 @@ import {
} from "@roo-code/types"
import { useAppTranslation } from "@src/i18n/TranslationContext"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, StandardTooltip } from "@src/components/ui"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@src/components/ui"
import { inputEventTransform, noTransform } from "../transforms"
import { inputEventTransform } from "../transforms"
type BedrockProps = {
apiConfiguration: ProviderSettings
setApiConfigurationField: (field: keyof ProviderSettings, value: ProviderSettings[keyof ProviderSettings]) => void
selectedModelInfo?: ModelInfo
simplifySettings?: boolean
}
export const Bedrock = ({ apiConfiguration, setApiConfigurationField, selectedModelInfo }: BedrockProps) => {
export const Bedrock = ({ apiConfiguration, setApiConfigurationField }: BedrockProps) => {
const { t } = useAppTranslation()
const [awsEndpointSelected, setAwsEndpointSelected] = useState(!!apiConfiguration?.awsBedrockEndpointEnabled)
@ -195,26 +193,6 @@ export const Bedrock = ({ apiConfiguration, setApiConfigurationField, selectedMo
}}>
{t("settings:providers.awsCrossRegion")}
</Checkbox>
{selectedModelInfo?.supportsPromptCache && (
<>
<Checkbox
checked={apiConfiguration?.awsUsePromptCache || false}
onChange={handleInputChange("awsUsePromptCache", noTransform)}>
<div className="flex items-center gap-1">
<span>{t("settings:providers.enablePromptCaching")}</span>
<StandardTooltip content={t("settings:providers.enablePromptCachingTitle")}>
<i
className="codicon codicon-info text-vscode-descriptionForeground"
style={{ fontSize: "12px" }}
/>
</StandardTooltip>
</div>
</Checkbox>
<div className="text-sm text-vscode-descriptionForeground ml-6 mt-1">
{t("settings:providers.cacheUsageNote")}
</div>
</>
)}
{supports1MContextBeta && (
<div>
<Checkbox

25
webview-ui/src/components/settings/providers/LiteLLM.tsx
View file

@ -1,5 +1,5 @@
import { useCallback, useState, useEffect, useRef } from "react"
import { VSCodeTextField, VSCodeCheckbox } from "@vscode/webview-ui-toolkit/react"
import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react"
import {
type ProviderSettings,
@ -159,29 +159,6 @@ export const LiteLLM = ({
errorMessage={modelValidationError}
simplifySettings={simplifySettings}
/>
{/* Show prompt caching option if the selected model supports it */}
{(() => {
const selectedModelId = apiConfiguration.litellmModelId || litellmDefaultModelId
const selectedModel = routerModels?.litellm?.[selectedModelId]
if (selectedModel?.supportsPromptCache) {
return (
<div className="mt-4">
<VSCodeCheckbox
checked={apiConfiguration.litellmUsePromptCache || false}
onChange={(e: any) => {
setApiConfigurationField("litellmUsePromptCache", e.target.checked)
}}>
<span className="font-medium">{t("settings:providers.enablePromptCaching")}</span>
</VSCodeCheckbox>
<div className="text-sm text-vscode-descriptionForeground ml-6 mt-1">
{t("settings:providers.enablePromptCachingTitle")}
</div>
</div>
)
}
return null
})()}
</>
)
}

16
webview-ui/src/components/settings/providers/__tests__/Bedrock.spec.tsx
View file

@ -260,6 +260,22 @@ describe("Bedrock Component", () => {
// Test Scenario 3: UI Elements Tests
describe("UI Elements", () => {
it("does not render legacy provider-level prompt caching controls", () => {
const apiConfiguration: Partial<ProviderSettings> = {
apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
awsUseProfile: true,
}
render(
<Bedrock
apiConfiguration={apiConfiguration as ProviderSettings}
setApiConfigurationField={mockSetApiConfigurationField}
/>,
)
expect(screen.queryByText("settings:providers.cacheUsageNote")).not.toBeInTheDocument()
})
it("should display example URLs when VPC endpoint checkbox is checked", () => {
const apiConfiguration: Partial<ProviderSettings> = {
awsBedrockEndpoint: "https://example.com",

3
webview-ui/src/context/ExtensionStateContext.tsx
View file

@ -124,8 +124,6 @@ export interface ExtensionStateContextType extends ExtensionState {
setTelemetrySetting: (value: TelemetrySetting) => void
remoteBrowserEnabled?: boolean
setRemoteBrowserEnabled: (value: boolean) => void
awsUsePromptCache?: boolean
setAwsUsePromptCache: (value: boolean) => void
maxImageFileSize: number
setMaxImageFileSize: (value: number) => void
maxTotalImageSize: number
@ -588,7 +586,6 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode
setShowRooIgnoredFiles: (value) => setState((prevState) => ({ ...prevState, showRooIgnoredFiles: value })),
setEnableSubfolderRules: (value) => setState((prevState) => ({ ...prevState, enableSubfolderRules: value })),
setRemoteBrowserEnabled: (value) => setState((prevState) => ({ ...prevState, remoteBrowserEnabled: value })),
setAwsUsePromptCache: (value) => setState((prevState) => ({ ...prevState, awsUsePromptCache: value })),
setMaxImageFileSize: (value) => setState((prevState) => ({ ...prevState, maxImageFileSize: value })),
setMaxTotalImageSize: (value) => setState((prevState) => ({ ...prevState, maxTotalImageSize: value })),
setPinnedApiConfigs: (value) => setState((prevState) => ({ ...prevState, pinnedApiConfigs: value })),