From 9099600363a1bda2b3e7f5352ce0937d0532822d Mon Sep 17 00:00:00 2001 From: Roo Code Date: Mon, 24 Nov 2025 22:17:20 +0000 Subject: [PATCH] fix: disable prompt caching for Azure OpenAI with GPT-5.1 models - Azure OpenAI does not support the prompt_cache_retention parameter - Added checks to skip prompt caching when Azure is detected - Fixed OpenAI provider to check for Azure before applying cache_control - Fixed OpenAI Native provider to skip prompt_cache_retention for Azure - Added tests to verify Azure OpenAI prompt caching is disabled Fixes #9544 --- .../openai-native-azure-prompt-cache.spec.ts | 252 ++++++++++++++++++ src/api/providers/openai-native.ts | 6 + src/api/providers/openai.ts | 11 +- 3 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 src/api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts diff --git a/src/api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts b/src/api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts new file mode 100644 index 0000000000..e7b03ad8fe --- /dev/null +++ b/src/api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts @@ -0,0 +1,252 @@ +// npx vitest run api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts + +import { OpenAiNativeHandler } from "../openai-native" +import { ApiHandlerOptions } from "../../../shared/api" +import { Anthropic } from "@anthropic-ai/sdk" + +// Mock the OpenAI SDK +const mockResponsesCreate = vitest.fn() + +vitest.mock("openai", () => ({ + __esModule: true, + default: vitest.fn().mockImplementation(() => ({ + responses: { + create: mockResponsesCreate.mockImplementation(async (requestBody) => { + // Return a mock async iterable for streaming + return { + [Symbol.asyncIterator]: async function* () { + yield { + type: "response.text.delta", + delta: "Test response", + } + yield { + type: "response.done", + response: { + usage: { + input_tokens: 100, + output_tokens: 50, + }, + }, + } + }, + } + }), + }, + })), +})) + +describe("OpenAiNativeHandler - Azure OpenAI Prompt Caching", () => { + let handler: OpenAiNativeHandler + let mockOptions: ApiHandlerOptions + + beforeEach(() => { + mockResponsesCreate.mockClear() + }) + + describe("Azure OpenAI with GPT-5.1 models", () => { + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: [ + { + type: "text" as const, + text: "Hello!", + }, + ], + }, + ] + + it("should NOT include prompt_cache_retention when using Azure OpenAI with GPT-5.1", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1", + openAiNativeBaseUrl: "https://myinstance.openai.azure.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // Should NOT have prompt_cache_retention because it's Azure + expect(callArgs).not.toHaveProperty("prompt_cache_retention") + }) + + it("should NOT include prompt_cache_retention when using Azure OpenAI with GPT-5.1-Codex", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1-codex", + openAiNativeBaseUrl: "https://myinstance.openai.azure.com/openai", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // Should NOT have prompt_cache_retention because it's Azure + expect(callArgs).not.toHaveProperty("prompt_cache_retention") + }) + + it("should NOT include prompt_cache_retention when Azure is detected via URL pattern", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1", + openAiNativeBaseUrl: "https://something.azure.com/openai", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // Should NOT have prompt_cache_retention because Azure is detected + expect(callArgs).not.toHaveProperty("prompt_cache_retention") + }) + + it("SHOULD include prompt_cache_retention=24h when NOT using Azure OpenAI with GPT-5.1", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1", + openAiNativeBaseUrl: "https://api.openai.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // SHOULD have prompt_cache_retention=24h when not Azure + expect(callArgs.prompt_cache_retention).toBe("24h") + }) + + it("SHOULD include prompt_cache_retention=24h when NOT using Azure OpenAI with GPT-5.1-Codex", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1-codex", + openAiNativeBaseUrl: "https://api.openai.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // SHOULD have prompt_cache_retention=24h when not Azure + expect(callArgs.prompt_cache_retention).toBe("24h") + }) + + it("should NOT include prompt_cache_retention for non-GPT-5.1 models regardless of Azure", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5", + openAiNativeBaseUrl: "https://api.openai.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // Should NOT have prompt_cache_retention for non-GPT-5.1 models + expect(callArgs).not.toHaveProperty("prompt_cache_retention") + }) + + it("should handle completePrompt without prompt_cache_retention when using Azure", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1", + openAiNativeBaseUrl: "https://myinstance.openai.azure.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + // Mock non-streaming response for completePrompt + mockResponsesCreate.mockResolvedValueOnce({ + output: [ + { + type: "message", + content: [ + { + type: "output_text", + text: "Test completion response", + }, + ], + }, + ], + }) + + const result = await handler.completePrompt("Test prompt") + + expect(result).toBe("Test completion response") + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // Should NOT have prompt_cache_retention because it's Azure + expect(callArgs).not.toHaveProperty("prompt_cache_retention") + }) + + it("should handle completePrompt with prompt_cache_retention=24h when NOT using Azure", async () => { + mockOptions = { + openAiNativeApiKey: "test-api-key", + apiModelId: "gpt-5.1", + openAiNativeBaseUrl: "https://api.openai.com", + } + handler = new OpenAiNativeHandler(mockOptions) + + // Mock non-streaming response for completePrompt + mockResponsesCreate.mockResolvedValueOnce({ + output: [ + { + type: "message", + content: [ + { + type: "output_text", + text: "Test completion response", + }, + ], + }, + ], + }) + + const result = await handler.completePrompt("Test prompt") + + expect(result).toBe("Test completion response") + expect(mockResponsesCreate).toHaveBeenCalled() + const callArgs = mockResponsesCreate.mock.calls[0][0] + + // SHOULD have prompt_cache_retention=24h when not Azure + expect(callArgs.prompt_cache_retention).toBe("24h") + }) + }) +}) diff --git a/src/api/providers/openai-native.ts b/src/api/providers/openai-native.ts index 74ba621c43..9508434bdc 100644 --- a/src/api/providers/openai-native.ts +++ b/src/api/providers/openai-native.ts @@ -1203,6 +1203,12 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio private getPromptCacheRetention(model: OpenAiNativeModel): "24h" | undefined { if (!model.info.supportsPromptCache) return undefined + // Azure OpenAI doesn't support prompt cache retention, so skip it + // Check if we're using Azure by looking at the base URL + const baseUrl = this.options.openAiNativeBaseUrl || "" + const isAzure = baseUrl.includes("azure.com") || baseUrl.includes(".openai.azure.com") + if (isAzure) return undefined + if (model.info.promptCacheRetention === "24h") { return "24h" } diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 1c8d3c7d9d..4aa7557645 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -94,6 +94,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl const deepseekReasoner = modelId.includes("deepseek-reasoner") || enabledR1Format const ark = modelUrl.includes(".volces.com") + // Check if we're using Azure OpenAI (same logic as in constructor) + const urlHost = this._getUrlHost(this.options.openAiBaseUrl) + const isAzureOpenAi = urlHost === "azure.com" || urlHost.endsWith(".azure.com") || this.options.openAiUseAzure + if (modelId.includes("o1") || modelId.includes("o3") || modelId.includes("o4")) { yield* this.handleO3FamilyMessage(modelId, systemPrompt, messages, metadata) return @@ -112,7 +116,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } else if (ark || enabledLegacyFormat) { convertedMessages = [systemMessage, ...convertToSimpleMessages(messages)] } else { - if (modelInfo.supportsPromptCache) { + // Azure OpenAI doesn't support prompt caching, so skip it when using Azure + const shouldUsePromptCache = modelInfo.supportsPromptCache && !isAzureOpenAi + + if (shouldUsePromptCache) { systemMessage = { role: "system", content: [ @@ -128,7 +135,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl convertedMessages = [systemMessage, ...convertToOpenAiMessages(messages)] - if (modelInfo.supportsPromptCache) { + if (shouldUsePromptCache) { // Note: the following logic is copied from openrouter: // Add cache_control to the last two user messages // (note: this works because we only ever add one user message at a time, but if we added multiple we'd need to mark the user message before the last assistant message)