mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-05 08:10:14 +00:00
fix: disable prompt caching for Azure OpenAI with GPT-5.1 models
- Azure OpenAI does not support the prompt_cache_retention parameter - Added checks to skip prompt caching when Azure is detected - Fixed OpenAI provider to check for Azure before applying cache_control - Fixed OpenAI Native provider to skip prompt_cache_retention for Azure - Added tests to verify Azure OpenAI prompt caching is disabled Fixes #9544
This commit is contained in:
Roo Code
parent
a81c4216c7
commit
9099600363
3 changed files with 267 additions and 2 deletions
|
|
@ -0,0 +1,252 @@
|
|||
// npx vitest run api/providers/__tests__/openai-native-azure-prompt-cache.spec.ts
|
||||
|
||||
import { OpenAiNativeHandler } from "../openai-native"
|
||||
import { ApiHandlerOptions } from "../../../shared/api"
|
||||
import { Anthropic } from "@anthropic-ai/sdk"
|
||||
|
||||
// Mock the OpenAI SDK
|
||||
const mockResponsesCreate = vitest.fn()
|
||||
|
||||
vitest.mock("openai", () => ({
|
||||
__esModule: true,
|
||||
default: vitest.fn().mockImplementation(() => ({
|
||||
responses: {
|
||||
create: mockResponsesCreate.mockImplementation(async (requestBody) => {
|
||||
// Return a mock async iterable for streaming
|
||||
return {
|
||||
[Symbol.asyncIterator]: async function* () {
|
||||
yield {
|
||||
type: "response.text.delta",
|
||||
delta: "Test response",
|
||||
}
|
||||
yield {
|
||||
type: "response.done",
|
||||
response: {
|
||||
usage: {
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}),
|
||||
},
|
||||
})),
|
||||
}))
|
||||
|
||||
describe("OpenAiNativeHandler - Azure OpenAI Prompt Caching", () => {
|
||||
let handler: OpenAiNativeHandler
|
||||
let mockOptions: ApiHandlerOptions
|
||||
|
||||
beforeEach(() => {
|
||||
mockResponsesCreate.mockClear()
|
||||
})
|
||||
|
||||
describe("Azure OpenAI with GPT-5.1 models", () => {
|
||||
const systemPrompt = "You are a helpful assistant."
|
||||
const messages: Anthropic.Messages.MessageParam[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text" as const,
|
||||
text: "Hello!",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
it("should NOT include prompt_cache_retention when using Azure OpenAI with GPT-5.1", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1",
|
||||
openAiNativeBaseUrl: "https://myinstance.openai.azure.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// Should NOT have prompt_cache_retention because it's Azure
|
||||
expect(callArgs).not.toHaveProperty("prompt_cache_retention")
|
||||
})
|
||||
|
||||
it("should NOT include prompt_cache_retention when using Azure OpenAI with GPT-5.1-Codex", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1-codex",
|
||||
openAiNativeBaseUrl: "https://myinstance.openai.azure.com/openai",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// Should NOT have prompt_cache_retention because it's Azure
|
||||
expect(callArgs).not.toHaveProperty("prompt_cache_retention")
|
||||
})
|
||||
|
||||
it("should NOT include prompt_cache_retention when Azure is detected via URL pattern", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1",
|
||||
openAiNativeBaseUrl: "https://something.azure.com/openai",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// Should NOT have prompt_cache_retention because Azure is detected
|
||||
expect(callArgs).not.toHaveProperty("prompt_cache_retention")
|
||||
})
|
||||
|
||||
it("SHOULD include prompt_cache_retention=24h when NOT using Azure OpenAI with GPT-5.1", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1",
|
||||
openAiNativeBaseUrl: "https://api.openai.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// SHOULD have prompt_cache_retention=24h when not Azure
|
||||
expect(callArgs.prompt_cache_retention).toBe("24h")
|
||||
})
|
||||
|
||||
it("SHOULD include prompt_cache_retention=24h when NOT using Azure OpenAI with GPT-5.1-Codex", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1-codex",
|
||||
openAiNativeBaseUrl: "https://api.openai.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// SHOULD have prompt_cache_retention=24h when not Azure
|
||||
expect(callArgs.prompt_cache_retention).toBe("24h")
|
||||
})
|
||||
|
||||
it("should NOT include prompt_cache_retention for non-GPT-5.1 models regardless of Azure", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5",
|
||||
openAiNativeBaseUrl: "https://api.openai.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
const stream = handler.createMessage(systemPrompt, messages)
|
||||
const chunks: any[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// Should NOT have prompt_cache_retention for non-GPT-5.1 models
|
||||
expect(callArgs).not.toHaveProperty("prompt_cache_retention")
|
||||
})
|
||||
|
||||
it("should handle completePrompt without prompt_cache_retention when using Azure", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1",
|
||||
openAiNativeBaseUrl: "https://myinstance.openai.azure.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
// Mock non-streaming response for completePrompt
|
||||
mockResponsesCreate.mockResolvedValueOnce({
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
content: [
|
||||
{
|
||||
type: "output_text",
|
||||
text: "Test completion response",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const result = await handler.completePrompt("Test prompt")
|
||||
|
||||
expect(result).toBe("Test completion response")
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// Should NOT have prompt_cache_retention because it's Azure
|
||||
expect(callArgs).not.toHaveProperty("prompt_cache_retention")
|
||||
})
|
||||
|
||||
it("should handle completePrompt with prompt_cache_retention=24h when NOT using Azure", async () => {
|
||||
mockOptions = {
|
||||
openAiNativeApiKey: "test-api-key",
|
||||
apiModelId: "gpt-5.1",
|
||||
openAiNativeBaseUrl: "https://api.openai.com",
|
||||
}
|
||||
handler = new OpenAiNativeHandler(mockOptions)
|
||||
|
||||
// Mock non-streaming response for completePrompt
|
||||
mockResponsesCreate.mockResolvedValueOnce({
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
content: [
|
||||
{
|
||||
type: "output_text",
|
||||
text: "Test completion response",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const result = await handler.completePrompt("Test prompt")
|
||||
|
||||
expect(result).toBe("Test completion response")
|
||||
expect(mockResponsesCreate).toHaveBeenCalled()
|
||||
const callArgs = mockResponsesCreate.mock.calls[0][0]
|
||||
|
||||
// SHOULD have prompt_cache_retention=24h when not Azure
|
||||
expect(callArgs.prompt_cache_retention).toBe("24h")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1203,6 +1203,12 @@ export class OpenAiNativeHandler extends BaseProvider implements SingleCompletio
|
|||
private getPromptCacheRetention(model: OpenAiNativeModel): "24h" | undefined {
|
||||
if (!model.info.supportsPromptCache) return undefined
|
||||
|
||||
// Azure OpenAI doesn't support prompt cache retention, so skip it
|
||||
// Check if we're using Azure by looking at the base URL
|
||||
const baseUrl = this.options.openAiNativeBaseUrl || ""
|
||||
const isAzure = baseUrl.includes("azure.com") || baseUrl.includes(".openai.azure.com")
|
||||
if (isAzure) return undefined
|
||||
|
||||
if (model.info.promptCacheRetention === "24h") {
|
||||
return "24h"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,6 +94,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
|
|||
const deepseekReasoner = modelId.includes("deepseek-reasoner") || enabledR1Format
|
||||
const ark = modelUrl.includes(".volces.com")
|
||||
|
||||
// Check if we're using Azure OpenAI (same logic as in constructor)
|
||||
const urlHost = this._getUrlHost(this.options.openAiBaseUrl)
|
||||
const isAzureOpenAi = urlHost === "azure.com" || urlHost.endsWith(".azure.com") || this.options.openAiUseAzure
|
||||
|
||||
if (modelId.includes("o1") || modelId.includes("o3") || modelId.includes("o4")) {
|
||||
yield* this.handleO3FamilyMessage(modelId, systemPrompt, messages, metadata)
|
||||
return
|
||||
|
|
@ -112,7 +116,10 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
|
|||
} else if (ark || enabledLegacyFormat) {
|
||||
convertedMessages = [systemMessage, ...convertToSimpleMessages(messages)]
|
||||
} else {
|
||||
if (modelInfo.supportsPromptCache) {
|
||||
// Azure OpenAI doesn't support prompt caching, so skip it when using Azure
|
||||
const shouldUsePromptCache = modelInfo.supportsPromptCache && !isAzureOpenAi
|
||||
|
||||
if (shouldUsePromptCache) {
|
||||
systemMessage = {
|
||||
role: "system",
|
||||
content: [
|
||||
|
|
@ -128,7 +135,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
|
|||
|
||||
convertedMessages = [systemMessage, ...convertToOpenAiMessages(messages)]
|
||||
|
||||
if (modelInfo.supportsPromptCache) {
|
||||
if (shouldUsePromptCache) {
|
||||
// Note: the following logic is copied from openrouter:
|
||||
// Add cache_control to the last two user messages
|
||||
// (note: this works because we only ever add one user message at a time, but if we added multiple we'd need to mark the user message before the last assistant message)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue