diff --git a/packages/types/src/providers/vercel-ai-gateway.ts b/packages/types/src/providers/vercel-ai-gateway.ts index 837baf7c52..bded0b530d 100644 --- a/packages/types/src/providers/vercel-ai-gateway.ts +++ b/packages/types/src/providers/vercel-ai-gateway.ts @@ -3,6 +3,29 @@ import type { ModelInfo } from "../model.js" // https://ai-gateway.vercel.sh/v1/ export const vercelAiGatewayDefaultModelId = "anthropic/claude-sonnet-4" +export const VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS = new Set([ + "anthropic/claude-3-haiku", + "anthropic/claude-3-opus", + "anthropic/claude-3.5-haiku", + "anthropic/claude-3.5-sonnet", + "anthropic/claude-3.7-sonnet", + "anthropic/claude-opus-4", + "anthropic/claude-opus-4.1", + "anthropic/claude-sonnet-4", + "openai/gpt-4.1", + "openai/gpt-4.1-mini", + "openai/gpt-4.1-nano", + "openai/gpt-4o", + "openai/gpt-4o-mini", + "openai/gpt-5", + "openai/gpt-5-mini", + "openai/gpt-5-nano", + "openai/o1", + "openai/o3", + "openai/o3-mini", + "openai/o4-mini", +]) + export const vercelAiGatewayDefaultModelInfo: ModelInfo = { maxTokens: 64000, contextWindow: 128000, diff --git a/src/api/providers/vercel-ai-gateway.ts b/src/api/providers/vercel-ai-gateway.ts index 87fb7bab9c..0b14d43c63 100644 --- a/src/api/providers/vercel-ai-gateway.ts +++ b/src/api/providers/vercel-ai-gateway.ts @@ -5,13 +5,14 @@ import { vercelAiGatewayDefaultModelId, vercelAiGatewayDefaultModelInfo, VERCEL_AI_GATEWAY_DEFAULT_TEMPERATURE, + VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS, } from "@roo-code/types" import { ApiHandlerOptions } from "../../shared/api" import { ApiStream } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" -import { addCacheBreakpoints } from "../transform/caching/anthropic" +import { addCacheBreakpoints } from "../transform/caching/vercel-ai-gateway" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" import { RouterProvider } from "./router-provider" @@ -49,9 +50,9 @@ export class VercelAiGatewayHandler extends RouterProvider implements SingleComp ...convertToOpenAiMessages(messages), ] - if (modelId.startsWith("anthropic/claude-3")) { + if (VERCEL_AI_GATEWAY_PROMPT_CACHING_MODELS.has(modelId) && info.supportsPromptCache) { addCacheBreakpoints(systemPrompt, openAiMessages) - } //TODO: add cache breakpoints for other models + } const body: OpenAI.Chat.ChatCompletionCreateParams = { model: modelId, diff --git a/src/api/transform/caching/vercel-ai-gateway.ts b/src/api/transform/caching/vercel-ai-gateway.ts new file mode 100644 index 0000000000..82eff0cd7b --- /dev/null +++ b/src/api/transform/caching/vercel-ai-gateway.ts @@ -0,0 +1,30 @@ +import OpenAI from "openai" + +export function addCacheBreakpoints(systemPrompt: string, messages: OpenAI.Chat.ChatCompletionMessageParam[]) { + // Apply cache_control to system message at the message level + messages[0] = { + role: "system", + content: systemPrompt, + // @ts-ignore-next-line + cache_control: { type: "ephemeral" }, + } + + // Add cache_control to the last two user messages for conversation context caching + const lastTwoUserMessages = messages.filter((msg) => msg.role === "user").slice(-2) + + lastTwoUserMessages.forEach((msg) => { + if (typeof msg.content === "string" && msg.content.length > 0) { + msg.content = [{ type: "text", text: msg.content }] + } + + if (Array.isArray(msg.content)) { + // Find the last text part in the message content + let lastTextPart = msg.content.filter((part) => part.type === "text").pop() + + if (lastTextPart && lastTextPart.text && lastTextPart.text.length > 0) { + // @ts-ignore-next-line + lastTextPart["cache_control"] = { type: "ephemeral" } + } + } + }) +} diff --git a/webview-ui/src/i18n/locales/de/settings.json b/webview-ui/src/i18n/locales/de/settings.json index 71006a9d4c..b9465d6c59 100644 --- a/webview-ui/src/i18n/locales/de/settings.json +++ b/webview-ui/src/i18n/locales/de/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Stellen Sie sicher, dass die Region in der ARN mit Ihrer oben ausgewählten AWS-Region übereinstimmt.", "openRouterApiKey": "OpenRouter API-Schlüssel", "getOpenRouterApiKey": "OpenRouter API-Schlüssel erhalten", + "vercelAiGatewayApiKey": "Vercel AI Gateway API-Schlüssel", "doubaoApiKey": "Doubao API-Schlüssel", "getDoubaoApiKey": "Doubao API-Schlüssel erhalten", "apiKeyStorageNotice": "API-Schlüssel werden sicher im VSCode Secret Storage gespeichert", diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 6d761e85ff..85bf310910 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -232,6 +232,7 @@ "awsCustomArnDesc": "Make sure the region in the ARN matches your selected AWS Region above.", "openRouterApiKey": "OpenRouter API Key", "getOpenRouterApiKey": "Get OpenRouter API Key", + "vercelAiGatewayApiKey": "Vercel AI Gateway API Key", "apiKeyStorageNotice": "API keys are stored securely in VSCode's Secret Storage", "glamaApiKey": "Glama API Key", "getGlamaApiKey": "Get Glama API Key", diff --git a/webview-ui/src/i18n/locales/es/settings.json b/webview-ui/src/i18n/locales/es/settings.json index a3dbfa410c..6eb3b5c3eb 100644 --- a/webview-ui/src/i18n/locales/es/settings.json +++ b/webview-ui/src/i18n/locales/es/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Asegúrese de que la región en el ARN coincida con la región de AWS seleccionada anteriormente.", "openRouterApiKey": "Clave API de OpenRouter", "getOpenRouterApiKey": "Obtener clave API de OpenRouter", + "vercelAiGatewayApiKey": "Clave API de Vercel AI Gateway", "apiKeyStorageNotice": "Las claves API se almacenan de forma segura en el Almacenamiento Secreto de VSCode", "glamaApiKey": "Clave API de Glama", "getGlamaApiKey": "Obtener clave API de Glama", diff --git a/webview-ui/src/i18n/locales/fr/settings.json b/webview-ui/src/i18n/locales/fr/settings.json index a2c2d0ca06..50124518d1 100644 --- a/webview-ui/src/i18n/locales/fr/settings.json +++ b/webview-ui/src/i18n/locales/fr/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Assurez-vous que la région dans l'ARN correspond à la région AWS sélectionnée ci-dessus.", "openRouterApiKey": "Clé API OpenRouter", "getOpenRouterApiKey": "Obtenir la clé API OpenRouter", + "vercelAiGatewayApiKey": "Clé API Vercel AI Gateway", "apiKeyStorageNotice": "Les clés API sont stockées en toute sécurité dans le stockage sécurisé de VSCode", "glamaApiKey": "Clé API Glama", "getGlamaApiKey": "Obtenir la clé API Glama", diff --git a/webview-ui/src/i18n/locales/it/settings.json b/webview-ui/src/i18n/locales/it/settings.json index 0cc2a59019..66ec621813 100644 --- a/webview-ui/src/i18n/locales/it/settings.json +++ b/webview-ui/src/i18n/locales/it/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Assicurati che la regione nell'ARN corrisponda alla regione AWS selezionata sopra.", "openRouterApiKey": "Chiave API OpenRouter", "getOpenRouterApiKey": "Ottieni chiave API OpenRouter", + "vercelAiGatewayApiKey": "Chiave API Vercel AI Gateway", "apiKeyStorageNotice": "Le chiavi API sono memorizzate in modo sicuro nell'Archivio Segreto di VSCode", "glamaApiKey": "Chiave API Glama", "getGlamaApiKey": "Ottieni chiave API Glama", diff --git a/webview-ui/src/i18n/locales/ja/settings.json b/webview-ui/src/i18n/locales/ja/settings.json index 27b51854dd..4e9b2848e2 100644 --- a/webview-ui/src/i18n/locales/ja/settings.json +++ b/webview-ui/src/i18n/locales/ja/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "ARN内のリージョンが上で選択したAWSリージョンと一致していることを確認してください。", "openRouterApiKey": "OpenRouter APIキー", "getOpenRouterApiKey": "OpenRouter APIキーを取得", + "vercelAiGatewayApiKey": "Vercel AI Gateway APIキー", "apiKeyStorageNotice": "APIキーはVSCodeのシークレットストレージに安全に保存されます", "glamaApiKey": "Glama APIキー", "getGlamaApiKey": "Glama APIキーを取得", diff --git a/webview-ui/src/i18n/locales/ko/settings.json b/webview-ui/src/i18n/locales/ko/settings.json index e6125f0ca8..2837b2dec3 100644 --- a/webview-ui/src/i18n/locales/ko/settings.json +++ b/webview-ui/src/i18n/locales/ko/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "ARN의 리전이 위에서 선택한 AWS 리전과 일치하는지 확인하세요.", "openRouterApiKey": "OpenRouter API 키", "getOpenRouterApiKey": "OpenRouter API 키 받기", + "vercelAiGatewayApiKey": "Vercel AI Gateway API 키", "apiKeyStorageNotice": "API 키는 VSCode의 보안 저장소에 안전하게 저장됩니다", "glamaApiKey": "Glama API 키", "getGlamaApiKey": "Glama API 키 받기", diff --git a/webview-ui/src/i18n/locales/pt-BR/settings.json b/webview-ui/src/i18n/locales/pt-BR/settings.json index 7039497b94..197c7af922 100644 --- a/webview-ui/src/i18n/locales/pt-BR/settings.json +++ b/webview-ui/src/i18n/locales/pt-BR/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Certifique-se de que a região no ARN corresponde à região AWS selecionada acima.", "openRouterApiKey": "Chave de API OpenRouter", "getOpenRouterApiKey": "Obter chave de API OpenRouter", + "vercelAiGatewayApiKey": "Chave API do Vercel AI Gateway", "apiKeyStorageNotice": "As chaves de API são armazenadas com segurança no Armazenamento Secreto do VSCode", "glamaApiKey": "Chave de API Glama", "getGlamaApiKey": "Obter chave de API Glama", diff --git a/webview-ui/src/i18n/locales/ru/settings.json b/webview-ui/src/i18n/locales/ru/settings.json index 05d73cd46f..a88dcfbc69 100644 --- a/webview-ui/src/i18n/locales/ru/settings.json +++ b/webview-ui/src/i18n/locales/ru/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "Убедитесь, что регион в ARN совпадает с выбранным выше регионом AWS.", "openRouterApiKey": "OpenRouter API-ключ", "getOpenRouterApiKey": "Получить OpenRouter API-ключ", + "vercelAiGatewayApiKey": "Ключ API Vercel AI Gateway", "apiKeyStorageNotice": "API-ключи хранятся безопасно в Secret Storage VSCode", "glamaApiKey": "Glama API-ключ", "getGlamaApiKey": "Получить Glama API-ключ", diff --git a/webview-ui/src/i18n/locales/zh-CN/settings.json b/webview-ui/src/i18n/locales/zh-CN/settings.json index a7d3077684..94b3db1885 100644 --- a/webview-ui/src/i18n/locales/zh-CN/settings.json +++ b/webview-ui/src/i18n/locales/zh-CN/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "请确保ARN中的区域与上方选择的AWS区域一致。", "openRouterApiKey": "OpenRouter API 密钥", "getOpenRouterApiKey": "获取 OpenRouter API 密钥", + "vercelAiGatewayApiKey": "Vercel AI Gateway API 密钥", "apiKeyStorageNotice": "API 密钥安全存储在 VSCode 的密钥存储中", "glamaApiKey": "Glama API 密钥", "getGlamaApiKey": "获取 Glama API 密钥", diff --git a/webview-ui/src/i18n/locales/zh-TW/settings.json b/webview-ui/src/i18n/locales/zh-TW/settings.json index d8b71601dc..31359291ce 100644 --- a/webview-ui/src/i18n/locales/zh-TW/settings.json +++ b/webview-ui/src/i18n/locales/zh-TW/settings.json @@ -233,6 +233,7 @@ "awsCustomArnDesc": "確保 ARN 中的區域與您上面選擇的 AWS 區域相符。", "openRouterApiKey": "OpenRouter API 金鑰", "getOpenRouterApiKey": "取得 OpenRouter API 金鑰", + "vercelAiGatewayApiKey": "Vercel AI Gateway API 金鑰", "apiKeyStorageNotice": "API 金鑰安全儲存於 VSCode 金鑰儲存中", "glamaApiKey": "Glama API 金鑰", "getGlamaApiKey": "取得 Glama API 金鑰",