From 95e4c1287310e8c3cf3d9a0bc0148d4bb62de0fa Mon Sep 17 00:00:00 2001 From: Roo Code Date: Tue, 23 Sep 2025 02:16:14 +0000 Subject: [PATCH] feat: separate OpenAI Chat Completions and Responses API handlers - Created base OpenAIBaseHandler with shared logic (auth, retry, metrics) - Split into OpenAIChatCompletionsHandler for Chat API - Split into OpenAIResponsesHandler for Responses API - OpenAiHandler now routes based on URL pattern and model type - Maintains backward compatibility with existing tests - Clear separation improves maintainability as APIs evolve Addresses #8246 --- .../__tests__/openai-architecture.spec.ts | 412 +++++++++++++++ src/api/providers/openai-base.ts | 137 +++++ src/api/providers/openai-chat-completions.ts | 382 ++++++++++++++ src/api/providers/openai-responses.ts | 489 ++++++++++++++++++ src/api/providers/openai.ts | 489 +++--------------- 5 files changed, 1494 insertions(+), 415 deletions(-) create mode 100644 src/api/providers/__tests__/openai-architecture.spec.ts create mode 100644 src/api/providers/openai-base.ts create mode 100644 src/api/providers/openai-chat-completions.ts create mode 100644 src/api/providers/openai-responses.ts diff --git a/src/api/providers/__tests__/openai-architecture.spec.ts b/src/api/providers/__tests__/openai-architecture.spec.ts new file mode 100644 index 0000000000..9dc7c6c8be --- /dev/null +++ b/src/api/providers/__tests__/openai-architecture.spec.ts @@ -0,0 +1,412 @@ +// npx vitest run api/providers/__tests__/openai-architecture.spec.ts + +import { OpenAiHandler } from "../openai" +import { OpenAIChatCompletionsHandler } from "../openai-chat-completions" +import { OpenAIResponsesHandler } from "../openai-responses" +import { ApiHandlerOptions } from "../../../shared/api" +import { Anthropic } from "@anthropic-ai/sdk" + +const mockCreate = vitest.fn() + +vitest.mock("openai", () => { + const mockConstructor = vitest.fn() + return { + __esModule: true, + default: mockConstructor.mockImplementation(() => ({ + chat: { + completions: { + create: mockCreate.mockImplementation(async (options) => { + if (!options.stream) { + return { + id: "test-completion", + choices: [ + { + message: { role: "assistant", content: "Test response", refusal: null }, + finish_reason: "stop", + index: 0, + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, + } + } + + return { + [Symbol.asyncIterator]: async function* () { + yield { + choices: [ + { + delta: { content: "Test response" }, + index: 0, + }, + ], + usage: null, + } + yield { + choices: [ + { + delta: {}, + index: 0, + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, + } + }, + } + }), + }, + }, + responses: { + create: vitest.fn().mockImplementation(async (options) => { + // Mock Responses API stream + return { + [Symbol.asyncIterator]: async function* () { + yield { + type: "response.text.delta", + delta: "Responses API response", + } + yield { + type: "response.done", + response: { + id: "resp_123", + usage: { + input_tokens: 10, + output_tokens: 5, + }, + }, + } + }, + } + }), + }, + })), + AzureOpenAI: mockConstructor.mockImplementation(() => ({ + chat: { + completions: { + create: mockCreate, + }, + }, + })), + } +}) + +describe("OpenAI Architecture Separation", () => { + let mockOptions: ApiHandlerOptions + + beforeEach(() => { + mockOptions = { + openAiApiKey: "test-api-key", + openAiModelId: "gpt-4", + openAiBaseUrl: "https://api.openai.com/v1", + } + mockCreate.mockClear() + }) + + describe("Handler Routing", () => { + it("should use Chat Completions handler by default", () => { + const handler = new OpenAiHandler(mockOptions) + expect(handler.getApiType()).toBe("chat-completions") + }) + + it("should use Responses API handler for GPT-5 models", () => { + const gpt5Options = { + ...mockOptions, + openAiModelId: "gpt-5-turbo", + } + const handler = new OpenAiHandler(gpt5Options) + expect(handler.getApiType()).toBe("responses") + }) + + it("should use Responses API handler when URL contains /v1/responses", () => { + const responsesOptions = { + ...mockOptions, + openAiBaseUrl: "https://api.openai.com/v1/responses", + } + const handler = new OpenAiHandler(responsesOptions) + expect(handler.getApiType()).toBe("responses") + }) + + it("should use Responses API handler when configured for OpenAI Native", () => { + const nativeOptions: ApiHandlerOptions = { + openAiNativeApiKey: "test-native-key", + openAiModelId: "gpt-4", + } + const handler = new OpenAiHandler(nativeOptions) + expect(handler.getApiType()).toBe("responses") + }) + + it("should use Chat Completions handler for standard OpenAI models", () => { + const handler = new OpenAiHandler(mockOptions) + expect(handler.getApiType()).toBe("chat-completions") + }) + + it("should use Chat Completions handler for O3 models", () => { + const o3Options = { + ...mockOptions, + openAiModelId: "o3-mini", + } + const handler = new OpenAiHandler(o3Options) + expect(handler.getApiType()).toBe("chat-completions") + }) + }) + + describe("Chat Completions Handler", () => { + it("should handle streaming messages correctly", async () => { + const handler = new OpenAIChatCompletionsHandler(mockOptions) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + ] + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(chunks.length).toBeGreaterThan(0) + const textChunks = chunks.filter((chunk) => chunk.type === "text") + expect(textChunks).toHaveLength(1) + expect(textChunks[0].text).toBe("Test response") + }) + + it("should handle O3 family models with special formatting", async () => { + const o3Options = { + ...mockOptions, + openAiModelId: "o3-mini", + } + const handler = new OpenAIChatCompletionsHandler(o3Options) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + ] + + const stream = handler.createMessage(systemPrompt, messages) + await stream.next() + + expect(mockCreate).toHaveBeenCalledWith( + expect.objectContaining({ + model: "o3-mini", + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "developer", + content: expect.stringContaining("Formatting re-enabled"), + }), + ]), + }), + {}, + ) + }) + + it("should handle DeepSeek reasoner models", async () => { + const deepseekOptions = { + ...mockOptions, + openAiModelId: "deepseek-reasoner", + } + const handler = new OpenAIChatCompletionsHandler(deepseekOptions) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + ] + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(chunks.length).toBeGreaterThan(0) + }) + }) + + describe("Responses API Handler", () => { + it("should handle streaming responses correctly", async () => { + const responsesOptions = { + ...mockOptions, + openAiBaseUrl: "https://api.openai.com/v1/responses", + } + const handler = new OpenAIResponsesHandler(responsesOptions) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + ] + + const stream = handler.createMessage(systemPrompt, messages) + const chunks: any[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + + expect(chunks.length).toBeGreaterThan(0) + const textChunks = chunks.filter((chunk) => chunk.type === "text") + expect(textChunks).toHaveLength(1) + expect(textChunks[0].text).toBe("Responses API response") + }) + + it("should maintain conversation continuity with response IDs", () => { + const handler = new OpenAIResponsesHandler(mockOptions) + + // Initially should have no response ID + expect(handler.getLastResponseId()).toBeUndefined() + + // Set a response ID + handler.setResponseId("resp_123") + expect(handler.getLastResponseId()).toBe("resp_123") + }) + + it("should format messages correctly for Responses API", async () => { + const handler = new OpenAIResponsesHandler(mockOptions) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + { + role: "assistant", + content: "Hi there!", + }, + { + role: "user", + content: "How are you?", + }, + ] + + const stream = handler.createMessage(systemPrompt, messages) + await stream.next() + + const responsesApi = (handler as any).client.responses + expect(responsesApi.create).toHaveBeenCalledWith( + expect.objectContaining({ + model: "gpt-4", + instructions: systemPrompt, + input: expect.arrayContaining([ + expect.objectContaining({ + role: "user", + content: expect.arrayContaining([ + expect.objectContaining({ + type: "input_text", + text: "Hello!", + }), + ]), + }), + ]), + }), + ) + }) + }) + + describe("Integration with Main Handler", () => { + it("should delegate to the correct handler based on configuration", async () => { + // Test Chat Completions delegation + const chatHandler = new OpenAiHandler(mockOptions) + const systemPrompt = "You are a helpful assistant." + const messages: Anthropic.Messages.MessageParam[] = [ + { + role: "user", + content: "Hello!", + }, + ] + + const chatStream = chatHandler.createMessage(systemPrompt, messages) + const chatChunks: any[] = [] + for await (const chunk of chatStream) { + chatChunks.push(chunk) + } + + expect(chatChunks.length).toBeGreaterThan(0) + expect(chatHandler.getApiType()).toBe("chat-completions") + + // Test Responses API delegation + const responsesOptions = { + ...mockOptions, + openAiModelId: "gpt-5-turbo", + } + const responsesHandler = new OpenAiHandler(responsesOptions) + + const responsesStream = responsesHandler.createMessage(systemPrompt, messages) + const responsesChunks: any[] = [] + for await (const chunk of responsesStream) { + responsesChunks.push(chunk) + } + + expect(responsesChunks.length).toBeGreaterThan(0) + expect(responsesHandler.getApiType()).toBe("responses") + }) + + it("should handle completePrompt through the correct handler", async () => { + const handler = new OpenAiHandler(mockOptions) + const result = await handler.completePrompt("Test prompt") + expect(result).toBe("Test response") + }) + + it("should expose response ID methods only for Responses API", () => { + // Chat Completions handler should return undefined + const chatHandler = new OpenAiHandler(mockOptions) + expect(chatHandler.getLastResponseId()).toBeUndefined() + chatHandler.setResponseId("test_id") // Should not throw but do nothing + expect(chatHandler.getLastResponseId()).toBeUndefined() + + // Responses API handler should work + const responsesOptions = { + ...mockOptions, + openAiModelId: "gpt-5-turbo", + } + const responsesHandler = new OpenAiHandler(responsesOptions) + responsesHandler.setResponseId("resp_456") + expect(responsesHandler.getLastResponseId()).toBe("resp_456") + }) + }) + + describe("Error Handling", () => { + it("should handle errors in Chat Completions handler", async () => { + mockCreate.mockRejectedValueOnce(new Error("API Error")) + const handler = new OpenAIChatCompletionsHandler(mockOptions) + + const stream = handler.createMessage("system", [{ role: "user", content: "test" }]) + + await expect(async () => { + for await (const _chunk of stream) { + // Should not reach here + } + }).rejects.toThrow("API Error") + }) + + it("should handle errors in Responses API handler", async () => { + const responsesApi = vitest.fn().mockRejectedValueOnce(new Error("Responses API Error")) + vitest.mocked((global as any).OpenAI).mockImplementationOnce(() => ({ + responses: { + create: responsesApi, + }, + })) + + const handler = new OpenAIResponsesHandler(mockOptions) + const stream = handler.createMessage("system", [{ role: "user", content: "test" }]) + + await expect(async () => { + for await (const _chunk of stream) { + // Should not reach here + } + }).rejects.toThrow() + }) + }) +}) diff --git a/src/api/providers/openai-base.ts b/src/api/providers/openai-base.ts new file mode 100644 index 0000000000..8525983241 --- /dev/null +++ b/src/api/providers/openai-base.ts @@ -0,0 +1,137 @@ +import { Anthropic } from "@anthropic-ai/sdk" +import OpenAI, { AzureOpenAI } from "openai" + +import { type ModelInfo, azureOpenAiDefaultApiVersion, openAiModelInfoSaneDefaults } from "@roo-code/types" + +import type { ApiHandlerOptions } from "../../shared/api" + +import { DEFAULT_HEADERS } from "./constants" +import { BaseProvider } from "./base-provider" +import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" +import { getApiRequestTimeout } from "./utils/timeout-config" +import { handleOpenAIError } from "./utils/openai-error-handler" +import { getModelParams } from "../transform/model-params" + +/** + * Base class for OpenAI-compatible providers with shared logic for auth, retry, metrics, and basic transforms. + * This class contains common functionality that both Chat Completions and Responses API handlers can use. + */ +export abstract class OpenAIBaseHandler extends BaseProvider implements SingleCompletionHandler { + protected options: ApiHandlerOptions + protected client: OpenAI + protected readonly providerName: string + + constructor(options: ApiHandlerOptions, providerName: string = "OpenAI") { + super() + this.options = options + this.providerName = providerName + + const baseURL = this.options.openAiBaseUrl ?? "https://api.openai.com/v1" + const apiKey = this.options.openAiApiKey ?? "not-provided" + const isAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) + const urlHost = this._getUrlHost(this.options.openAiBaseUrl) + const isAzureOpenAi = urlHost === "azure.com" || urlHost.endsWith(".azure.com") || options.openAiUseAzure + + const headers = { + ...DEFAULT_HEADERS, + ...(this.options.openAiHeaders || {}), + } + + const timeout = getApiRequestTimeout() + + if (isAzureAiInference) { + // Azure AI Inference Service (e.g., for DeepSeek) uses a different path structure + this.client = new OpenAI({ + baseURL, + apiKey, + defaultHeaders: headers, + defaultQuery: { "api-version": this.options.azureApiVersion || "2024-05-01-preview" }, + timeout, + }) + } else if (isAzureOpenAi) { + // Azure API shape slightly differs from the core API shape: + // https://github.com/openai/openai-node?tab=readme-ov-file#microsoft-azure-openai + this.client = new AzureOpenAI({ + baseURL, + apiKey, + apiVersion: this.options.azureApiVersion || azureOpenAiDefaultApiVersion, + defaultHeaders: headers, + timeout, + }) + } else { + this.client = new OpenAI({ + baseURL, + apiKey, + defaultHeaders: headers, + timeout, + }) + } + } + + /** + * Get the model configuration with parameters + */ + override getModel() { + const id = this.options.openAiModelId ?? "" + const info = this.options.openAiCustomModelInfo ?? openAiModelInfoSaneDefaults + const params = getModelParams({ format: "openai", modelId: id, model: info, settings: this.options }) + return { id, info, ...params } + } + + /** + * Complete a prompt using the appropriate API + */ + abstract completePrompt(prompt: string): Promise + + /** + * Helper method to get the URL host + */ + protected _getUrlHost(baseUrl?: string): string { + try { + return new URL(baseUrl ?? "").host + } catch (error) { + return "" + } + } + + /** + * Check if the URL is for Grok xAI + */ + protected _isGrokXAI(baseUrl?: string): boolean { + const urlHost = this._getUrlHost(baseUrl) + return urlHost.includes("x.ai") + } + + /** + * Check if the URL is for Azure AI Inference + */ + protected _isAzureAiInference(baseUrl?: string): boolean { + const urlHost = this._getUrlHost(baseUrl) + return urlHost.endsWith(".services.ai.azure.com") + } + + /** + * Handle OpenAI errors with proper error messages + */ + protected handleError(error: any): never { + throw handleOpenAIError(error, this.providerName) + } + + /** + * Adds max_completion_tokens to the request body if needed based on provider configuration + * Note: max_tokens is deprecated in favor of max_completion_tokens as per OpenAI documentation + */ + protected addMaxTokensIfNeeded( + requestOptions: + | OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming + | OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, + modelInfo: ModelInfo, + ): void { + // Only add max_completion_tokens if includeMaxTokens is true + if (this.options.includeMaxTokens === true) { + // Use user-configured modelMaxTokens if available, otherwise fall back to model's default maxTokens + // Using max_completion_tokens as max_tokens is deprecated + requestOptions.max_completion_tokens = this.options.modelMaxTokens || modelInfo.maxTokens + } + } +} diff --git a/src/api/providers/openai-chat-completions.ts b/src/api/providers/openai-chat-completions.ts new file mode 100644 index 0000000000..f7d060aa60 --- /dev/null +++ b/src/api/providers/openai-chat-completions.ts @@ -0,0 +1,382 @@ +import { Anthropic } from "@anthropic-ai/sdk" +import OpenAI from "openai" + +import { type ModelInfo, DEEP_SEEK_DEFAULT_TEMPERATURE, OPENAI_AZURE_AI_INFERENCE_PATH } from "@roo-code/types" + +import type { ApiHandlerOptions } from "../../shared/api" + +import { XmlMatcher } from "../../utils/xml-matcher" + +import { convertToOpenAiMessages } from "../transform/openai-format" +import { convertToR1Format } from "../transform/r1-format" +import { convertToSimpleMessages } from "../transform/simple-format" +import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" + +import { OpenAIBaseHandler } from "./openai-base" +import type { ApiHandlerCreateMessageMetadata } from "../index" + +/** + * Handler for OpenAI Chat Completions API + * Handles standard chat completions, o1/o3/o4 models, DeepSeek reasoner, and other variations + */ +export class OpenAIChatCompletionsHandler extends OpenAIBaseHandler { + constructor(options: ApiHandlerOptions) { + // Use "OpenAI" for backward compatibility with existing tests + super(options, "OpenAI") + } + + override async *createMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): ApiStream { + const { info: modelInfo, reasoning } = this.getModel() + const modelUrl = this.options.openAiBaseUrl ?? "" + const modelId = this.options.openAiModelId ?? "" + const enabledR1Format = this.options.openAiR1FormatEnabled ?? false + const enabledLegacyFormat = this.options.openAiLegacyFormat ?? false + const isAzureAiInference = this._isAzureAiInference(modelUrl) + const deepseekReasoner = modelId.includes("deepseek-reasoner") || enabledR1Format + const ark = modelUrl.includes(".volces.com") + + // Handle O1/O3/O4 family models separately + if (modelId.includes("o1") || modelId.includes("o3") || modelId.includes("o4")) { + yield* this.handleO3FamilyMessage(modelId, systemPrompt, messages) + return + } + + if (this.options.openAiStreamingEnabled ?? true) { + yield* this.handleStreamingMessage( + systemPrompt, + messages, + modelInfo, + modelId, + deepseekReasoner, + ark, + enabledLegacyFormat, + isAzureAiInference, + reasoning, + ) + } else { + yield* this.handleNonStreamingMessage( + systemPrompt, + messages, + modelInfo, + modelId, + deepseekReasoner, + enabledLegacyFormat, + isAzureAiInference, + ) + } + } + + private async *handleStreamingMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelInfo: ModelInfo, + modelId: string, + deepseekReasoner: boolean, + ark: boolean, + enabledLegacyFormat: boolean, + isAzureAiInference: boolean, + reasoning: any, + ): ApiStream { + let systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = { + role: "system", + content: systemPrompt, + } + + let convertedMessages + + if (deepseekReasoner) { + convertedMessages = convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) + } else if (ark || enabledLegacyFormat) { + convertedMessages = [systemMessage, ...convertToSimpleMessages(messages)] + } else { + if (modelInfo.supportsPromptCache) { + systemMessage = { + role: "system", + content: [ + { + type: "text", + text: systemPrompt, + // @ts-ignore-next-line + cache_control: { type: "ephemeral" }, + }, + ], + } + } + + convertedMessages = [systemMessage, ...convertToOpenAiMessages(messages)] + + if (modelInfo.supportsPromptCache) { + // Add cache_control to the last two user messages + const lastTwoUserMessages = convertedMessages.filter((msg) => msg.role === "user").slice(-2) + + lastTwoUserMessages.forEach((msg) => { + if (typeof msg.content === "string") { + msg.content = [{ type: "text", text: msg.content }] + } + + if (Array.isArray(msg.content)) { + let lastTextPart = msg.content.filter((part) => part.type === "text").pop() + + if (!lastTextPart) { + lastTextPart = { type: "text", text: "..." } + msg.content.push(lastTextPart) + } + + // @ts-ignore-next-line + lastTextPart["cache_control"] = { type: "ephemeral" } + } + }) + } + } + + const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) + + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + model: modelId, + temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0), + messages: convertedMessages, + stream: true as const, + ...(isGrokXAI ? {} : { stream_options: { include_usage: true } }), + ...(reasoning && reasoning), + } + + // Add max_tokens if needed + this.addMaxTokensIfNeeded(requestOptions, modelInfo) + + let stream + try { + stream = await this.client.chat.completions.create( + requestOptions, + isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, + ) + } catch (error) { + this.handleError(error) + } + + const matcher = new XmlMatcher( + "think", + (chunk) => + ({ + type: chunk.matched ? "reasoning" : "text", + text: chunk.data, + }) as const, + ) + + let lastUsage + + for await (const chunk of stream) { + const delta = chunk.choices[0]?.delta ?? {} + + if (delta.content) { + for (const chunk of matcher.update(delta.content)) { + yield chunk + } + } + + if ("reasoning_content" in delta && delta.reasoning_content) { + yield { + type: "reasoning", + text: (delta.reasoning_content as string | undefined) || "", + } + } + if (chunk.usage) { + lastUsage = chunk.usage + } + } + + for (const chunk of matcher.final()) { + yield chunk + } + + if (lastUsage) { + yield this.processUsageMetrics(lastUsage, modelInfo) + } + } + + private async *handleNonStreamingMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + modelInfo: ModelInfo, + modelId: string, + deepseekReasoner: boolean, + enabledLegacyFormat: boolean, + isAzureAiInference: boolean, + ): ApiStream { + const systemMessage: OpenAI.Chat.ChatCompletionUserMessageParam = { + role: "user", + content: systemPrompt, + } + + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + model: modelId, + messages: deepseekReasoner + ? convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) + : enabledLegacyFormat + ? [systemMessage, ...convertToSimpleMessages(messages)] + : [systemMessage, ...convertToOpenAiMessages(messages)], + } + + // Add max_tokens if needed + this.addMaxTokensIfNeeded(requestOptions, modelInfo) + + let response + try { + response = await this.client.chat.completions.create( + requestOptions, + isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, + ) + } catch (error) { + this.handleError(error) + } + + yield { + type: "text", + text: response.choices[0]?.message.content || "", + } + + yield this.processUsageMetrics(response.usage, modelInfo) + } + + private async *handleO3FamilyMessage( + modelId: string, + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + ): ApiStream { + const modelInfo = this.getModel().info + const methodIsAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) + + if (this.options.openAiStreamingEnabled ?? true) { + const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) + + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { + model: modelId, + messages: [ + { + role: "developer", + content: `Formatting re-enabled\n${systemPrompt}`, + }, + ...convertToOpenAiMessages(messages), + ], + stream: true, + ...(isGrokXAI ? {} : { stream_options: { include_usage: true } }), + reasoning_effort: modelInfo.reasoningEffort as "low" | "medium" | "high" | undefined, + temperature: undefined, + } + + // O3 family models support max_completion_tokens + this.addMaxTokensIfNeeded(requestOptions, modelInfo) + + let stream + try { + stream = await this.client.chat.completions.create( + requestOptions, + methodIsAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, + ) + } catch (error) { + this.handleError(error) + } + + yield* this.handleStreamResponse(stream) + } else { + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + model: modelId, + messages: [ + { + role: "developer", + content: `Formatting re-enabled\n${systemPrompt}`, + }, + ...convertToOpenAiMessages(messages), + ], + reasoning_effort: modelInfo.reasoningEffort as "low" | "medium" | "high" | undefined, + temperature: undefined, + } + + // O3 family models support max_completion_tokens + this.addMaxTokensIfNeeded(requestOptions, modelInfo) + + let response + try { + response = await this.client.chat.completions.create( + requestOptions, + methodIsAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, + ) + } catch (error) { + this.handleError(error) + } + + yield { + type: "text", + text: response.choices[0]?.message.content || "", + } + yield this.processUsageMetrics(response.usage) + } + } + + private async *handleStreamResponse(stream: AsyncIterable): ApiStream { + for await (const chunk of stream) { + const delta = chunk.choices[0]?.delta + if (delta?.content) { + yield { + type: "text", + text: delta.content, + } + } + + if (chunk.usage) { + yield { + type: "usage", + inputTokens: chunk.usage.prompt_tokens || 0, + outputTokens: chunk.usage.completion_tokens || 0, + } + } + } + } + + protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { + return { + type: "usage", + inputTokens: usage?.prompt_tokens || 0, + outputTokens: usage?.completion_tokens || 0, + cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, + cacheReadTokens: usage?.cache_read_input_tokens || undefined, + } + } + + async completePrompt(prompt: string): Promise { + try { + const isAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) + const model = this.getModel() + const modelInfo = model.info + + const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { + model: model.id, + messages: [{ role: "user", content: prompt }], + } + + // Add max_tokens if needed + this.addMaxTokensIfNeeded(requestOptions, modelInfo) + + let response + try { + response = await this.client.chat.completions.create( + requestOptions, + isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, + ) + } catch (error) { + this.handleError(error) + } + + return response.choices[0]?.message.content || "" + } catch (error) { + if (error instanceof Error) { + throw new Error(`${this.providerName} completion error: ${error.message}`) + } + + throw error + } + } +} diff --git a/src/api/providers/openai-responses.ts b/src/api/providers/openai-responses.ts new file mode 100644 index 0000000000..e36a31e439 --- /dev/null +++ b/src/api/providers/openai-responses.ts @@ -0,0 +1,489 @@ +import { Anthropic } from "@anthropic-ai/sdk" + +import { + type ModelInfo, + type ServiceTier, + OPENAI_NATIVE_DEFAULT_TEMPERATURE, + GPT5_DEFAULT_TEMPERATURE, +} from "@roo-code/types" + +import type { ApiHandlerOptions } from "../../shared/api" + +import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" +import { calculateApiCostOpenAI } from "../../shared/cost" + +import { OpenAIBaseHandler } from "./openai-base" +import type { ApiHandlerCreateMessageMetadata } from "../index" + +/** + * Handler for OpenAI Responses API + * Handles the new Responses API with specialized streaming/response parsing + * This is a simplified implementation focusing on the core Responses API functionality + */ +export class OpenAIResponsesHandler extends OpenAIBaseHandler { + private lastResponseId: string | undefined + private lastServiceTier: ServiceTier | undefined + + constructor(options: ApiHandlerOptions) { + super(options, "OpenAI Responses") + } + + override async *createMessage( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): ApiStream { + // Reset resolved tier for this request + this.lastServiceTier = undefined + + const model = this.getModel() + const modelId = model.id + + // Prepare the request body for Responses API + const requestBody = this.buildRequestBody(model, systemPrompt, messages, metadata) + + // Make the request + yield* this.executeRequest(requestBody, model) + } + + private buildRequestBody( + model: any, + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): any { + // Format the conversation for Responses API + const formattedInput = this.formatConversation(systemPrompt, messages, metadata) + + const requestBody: any = { + model: model.id, + input: formattedInput, + stream: true, + store: metadata?.store !== false, // Default to true unless explicitly set to false + instructions: systemPrompt, + } + + // Add temperature if supported + if (model.info.supportsTemperature !== false) { + requestBody.temperature = + this.options.modelTemperature ?? + (model.id.startsWith("gpt-5") ? GPT5_DEFAULT_TEMPERATURE : OPENAI_NATIVE_DEFAULT_TEMPERATURE) + } + + // Add max output tokens if available + if (model.maxTokens) { + requestBody.max_output_tokens = model.maxTokens + } + + // Add previous response ID if available for conversation continuity + if (this.lastResponseId && !metadata?.suppressPreviousResponseId) { + requestBody.previous_response_id = this.lastResponseId + } + + return requestBody + } + + private formatConversation( + systemPrompt: string, + messages: Anthropic.Messages.MessageParam[], + metadata?: ApiHandlerCreateMessageMetadata, + ): any[] { + // If we have a previous response ID and not suppressed, only send the latest user message + if (this.lastResponseId && !metadata?.suppressPreviousResponseId) { + const lastUserMessage = [...messages].reverse().find((msg) => msg.role === "user") + if (lastUserMessage) { + return [this.formatMessage(lastUserMessage)] + } + } + + // Otherwise, format the full conversation + const formattedMessages: any[] = [] + + for (const message of messages) { + formattedMessages.push(this.formatMessage(message)) + } + + return formattedMessages + } + + private formatMessage(message: Anthropic.Messages.MessageParam): any { + const role = message.role === "user" ? "user" : "assistant" + const content: any[] = [] + + if (typeof message.content === "string") { + // Use input_text for user messages, output_text for assistant + if (role === "user") { + content.push({ type: "input_text", text: message.content }) + } else { + content.push({ type: "output_text", text: message.content }) + } + } else if (Array.isArray(message.content)) { + for (const block of message.content) { + if (block.type === "text") { + if (role === "user") { + content.push({ type: "input_text", text: (block as any).text }) + } else { + content.push({ type: "output_text", text: (block as any).text }) + } + } else if (block.type === "image") { + const image = block as Anthropic.Messages.ImageBlockParam + const imageUrl = `data:${image.source.media_type};base64,${image.source.data}` + content.push({ type: "input_image", image_url: imageUrl }) + } + } + } + + return { role, content } + } + + private async *executeRequest(requestBody: any, model: any): ApiStream { + try { + // Use the SDK's Responses API if available + const responsesApi = (this.client as any).responses + if (responsesApi && typeof responsesApi.create === "function") { + const stream = await responsesApi.create(requestBody) + + if (typeof (stream as any)[Symbol.asyncIterator] === "function") { + for await (const event of stream) { + yield* this.processEvent(event, model) + } + return + } + } + + // Fallback to direct API call if SDK doesn't support Responses API + yield* this.makeDirectApiCall(requestBody, model) + } catch (error: any) { + // Handle previous_response_id not found error + if (error?.status === 400 && requestBody.previous_response_id) { + // Clear the stored lastResponseId and retry without it + this.lastResponseId = undefined + delete requestBody.previous_response_id + + // Retry the request + try { + const responsesApi = (this.client as any).responses + if (responsesApi && typeof responsesApi.create === "function") { + const stream = await responsesApi.create(requestBody) + if (typeof (stream as any)[Symbol.asyncIterator] === "function") { + for await (const event of stream) { + yield* this.processEvent(event, model) + } + return + } + } + yield* this.makeDirectApiCall(requestBody, model) + } catch (retryError) { + this.handleError(retryError) + } + } else { + this.handleError(error) + } + } + } + + private async *makeDirectApiCall(requestBody: any, model: any): ApiStream { + const apiKey = this.options.openAiApiKey ?? "not-provided" + const baseUrl = this.options.openAiBaseUrl || "https://api.openai.com" + const url = `${baseUrl}/v1/responses` + + try { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + Accept: "text/event-stream", + }, + body: JSON.stringify(requestBody), + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`Responses API error (${response.status}): ${errorText}`) + } + + if (!response.body) { + throw new Error("Responses API error: No response body") + } + + // Handle streaming response + yield* this.handleStreamResponse(response.body, model) + } catch (error) { + this.handleError(error) + } + } + + private async *handleStreamResponse(body: ReadableStream, model: any): ApiStream { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = "" + + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() || "" + + for (const line of lines) { + if (line.startsWith("data: ")) { + const data = line.slice(6).trim() + if (data === "[DONE]") { + continue + } + + try { + const parsed = JSON.parse(data) + yield* this.processEvent(parsed, model) + } catch (e) { + // Ignore JSON parsing errors + if (!(e instanceof SyntaxError)) { + throw e + } + } + } + } + } + } finally { + reader.releaseLock() + } + } + + private async *processEvent(event: any, model: any): ApiStream { + // Store response ID for conversation continuity + if (event?.response?.id) { + this.lastResponseId = event.response.id + } + + // Capture resolved service tier + if (event?.response?.service_tier) { + this.lastServiceTier = event.response.service_tier as ServiceTier + } + + // Handle text deltas + if (event?.type === "response.text.delta" || event?.type === "response.output_text.delta") { + if (event?.delta) { + yield { type: "text", text: event.delta } + } + return + } + + // Handle reasoning deltas + if ( + event?.type === "response.reasoning.delta" || + event?.type === "response.reasoning_text.delta" || + event?.type === "response.reasoning_summary.delta" || + event?.type === "response.reasoning_summary_text.delta" + ) { + if (event?.delta) { + yield { type: "reasoning", text: event.delta } + } + return + } + + // Handle refusal deltas + if (event?.type === "response.refusal.delta") { + if (event?.delta) { + yield { type: "text", text: `[Refusal] ${event.delta}` } + } + return + } + + // Handle output item additions + if (event?.type === "response.output_item.added") { + const item = event?.item + if (item) { + if (item.type === "text" && item.text) { + yield { type: "text", text: item.text } + } else if (item.type === "reasoning" && item.text) { + yield { type: "reasoning", text: item.text } + } else if (item.type === "message" && Array.isArray(item.content)) { + for (const content of item.content) { + if ((content?.type === "text" || content?.type === "output_text") && content?.text) { + yield { type: "text", text: content.text } + } + } + } + } + return + } + + // Handle completion events with usage + if (event?.type === "response.done" || event?.type === "response.completed") { + const usage = event?.response?.usage || event?.usage + if (usage) { + yield this.normalizeUsage(usage, model) + } + return + } + + // Handle complete response in initial event (non-streaming format) + if (event.response && event.response.output && Array.isArray(event.response.output)) { + for (const outputItem of event.response.output) { + if (outputItem.type === "text" && outputItem.content) { + for (const content of outputItem.content) { + if (content.type === "text" && content.text) { + yield { type: "text", text: content.text } + } + } + } + // Handle reasoning summaries + if (outputItem.type === "reasoning" && Array.isArray(outputItem.summary)) { + for (const summary of outputItem.summary) { + if (summary?.type === "summary_text" && typeof summary.text === "string") { + yield { type: "reasoning", text: summary.text } + } + } + } + } + // Check for usage in the complete response + if (event.response.usage) { + yield this.normalizeUsage(event.response.usage, model) + } + } + } + + private normalizeUsage(usage: any, model: any): ApiStreamUsageChunk { + if (!usage) { + return { + type: "usage", + inputTokens: 0, + outputTokens: 0, + } + } + + const inputDetails = usage.input_tokens_details ?? usage.prompt_tokens_details + const cachedFromDetails = inputDetails?.cached_tokens ?? 0 + const missFromDetails = inputDetails?.cache_miss_tokens ?? 0 + + let totalInputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0 + if (totalInputTokens === 0 && inputDetails && (cachedFromDetails > 0 || missFromDetails > 0)) { + totalInputTokens = cachedFromDetails + missFromDetails + } + + const totalOutputTokens = usage.output_tokens ?? usage.completion_tokens ?? 0 + const cacheWriteTokens = usage.cache_creation_input_tokens ?? usage.cache_write_tokens ?? 0 + const cacheReadTokens = + usage.cache_read_input_tokens ?? usage.cache_read_tokens ?? usage.cached_tokens ?? cachedFromDetails ?? 0 + + // Calculate cost if we have pricing info + const effectiveTier = this.lastServiceTier || undefined + const effectiveInfo = this.applyServiceTierPricing(model.info, effectiveTier) + + const totalCost = calculateApiCostOpenAI( + effectiveInfo, + totalInputTokens, + totalOutputTokens, + cacheWriteTokens, + cacheReadTokens, + ) + + const reasoningTokens = + typeof usage.output_tokens_details?.reasoning_tokens === "number" + ? usage.output_tokens_details.reasoning_tokens + : undefined + + return { + type: "usage", + inputTokens: totalInputTokens, + outputTokens: totalOutputTokens, + cacheWriteTokens, + cacheReadTokens, + ...(typeof reasoningTokens === "number" ? { reasoningTokens } : {}), + totalCost, + } + } + + private applyServiceTierPricing(info: ModelInfo, tier?: ServiceTier): ModelInfo { + if (!tier || tier === "default") return info + + const tierInfo = info.tiers?.find((t) => t.name === tier) + if (!tierInfo) return info + + return { + ...info, + inputPrice: tierInfo.inputPrice ?? info.inputPrice, + outputPrice: tierInfo.outputPrice ?? info.outputPrice, + cacheReadsPrice: tierInfo.cacheReadsPrice ?? info.cacheReadsPrice, + cacheWritesPrice: tierInfo.cacheWritesPrice ?? info.cacheWritesPrice, + } + } + + async completePrompt(prompt: string): Promise { + try { + const model = this.getModel() + + const requestBody: any = { + model: model.id, + input: [ + { + role: "user", + content: [{ type: "input_text", text: prompt }], + }, + ], + stream: false, + store: false, + } + + // Add temperature if supported + if (model.info.supportsTemperature !== false) { + requestBody.temperature = + this.options.modelTemperature ?? + (model.id.startsWith("gpt-5") ? GPT5_DEFAULT_TEMPERATURE : OPENAI_NATIVE_DEFAULT_TEMPERATURE) + } + + // Add max output tokens if available + if (model.maxTokens) { + requestBody.max_output_tokens = model.maxTokens + } + + // Make the non-streaming request + const responsesApi = (this.client as any).responses + if (responsesApi && typeof responsesApi.create === "function") { + const response = await responsesApi.create(requestBody) + + // Extract text from the response + if (response?.output && Array.isArray(response.output)) { + for (const outputItem of response.output) { + if (outputItem.type === "message" && outputItem.content) { + for (const content of outputItem.content) { + if (content.type === "output_text" && content.text) { + return content.text + } + } + } + } + } + + // Fallback: check for direct text in response + if (response?.text) { + return response.text + } + } + + return "" + } catch (error) { + if (error instanceof Error) { + throw new Error(`${this.providerName} completion error: ${error.message}`) + } + throw error + } + } + + /** + * Gets the last response ID for conversation continuity + */ + getLastResponseId(): string | undefined { + return this.lastResponseId + } + + /** + * Sets the response ID for conversation continuity + */ + setResponseId(responseId: string): void { + this.lastResponseId = responseId + } +} diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index aebe671712..5dc7412db1 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -1,444 +1,103 @@ import { Anthropic } from "@anthropic-ai/sdk" -import OpenAI, { AzureOpenAI } from "openai" import axios from "axios" -import { - type ModelInfo, - azureOpenAiDefaultApiVersion, - openAiModelInfoSaneDefaults, - DEEP_SEEK_DEFAULT_TEMPERATURE, - OPENAI_AZURE_AI_INFERENCE_PATH, -} from "@roo-code/types" - -import type { ApiHandlerOptions } from "../../shared/api" - -import { XmlMatcher } from "../../utils/xml-matcher" - -import { convertToOpenAiMessages } from "../transform/openai-format" -import { convertToR1Format } from "../transform/r1-format" -import { convertToSimpleMessages } from "../transform/simple-format" -import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" -import { getModelParams } from "../transform/model-params" - import { DEFAULT_HEADERS } from "./constants" -import { BaseProvider } from "./base-provider" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" -import { getApiRequestTimeout } from "./utils/timeout-config" -import { handleOpenAIError } from "./utils/openai-error-handler" +import type { ApiHandlerOptions } from "../../shared/api" +import { ApiStream } from "../transform/stream" -// TODO: Rename this to OpenAICompatibleHandler. Also, I think the -// `OpenAINativeHandler` can subclass from this, since it's obviously -// compatible with the OpenAI API. We can also rename it to `OpenAIHandler`. -export class OpenAiHandler extends BaseProvider implements SingleCompletionHandler { - protected options: ApiHandlerOptions - private client: OpenAI - private readonly providerName = "OpenAI" +import { OpenAIChatCompletionsHandler } from "./openai-chat-completions" +import { OpenAIResponsesHandler } from "./openai-responses" + +/** + * Main OpenAI handler that routes requests to either Chat Completions or Responses API + * based on URL pattern detection and configuration. + * + * This handler acts as a router, automatically detecting which API to use based on: + * - URL patterns (e.g., /v1/responses for Responses API) + * - Model configuration (e.g., GPT-5 models prefer Responses API) + * - Explicit configuration flags + */ +export class OpenAiHandler implements SingleCompletionHandler { + private handler: OpenAIChatCompletionsHandler | OpenAIResponsesHandler + private readonly useResponsesApi: boolean constructor(options: ApiHandlerOptions) { - super() - this.options = options + // Determine which API to use based on configuration and URL patterns + this.useResponsesApi = this.shouldUseResponsesApi(options) - const baseURL = this.options.openAiBaseUrl ?? "https://api.openai.com/v1" - const apiKey = this.options.openAiApiKey ?? "not-provided" - const isAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) - const urlHost = this._getUrlHost(this.options.openAiBaseUrl) - const isAzureOpenAi = urlHost === "azure.com" || urlHost.endsWith(".azure.com") || options.openAiUseAzure - - const headers = { - ...DEFAULT_HEADERS, - ...(this.options.openAiHeaders || {}), - } - - const timeout = getApiRequestTimeout() - - if (isAzureAiInference) { - // Azure AI Inference Service (e.g., for DeepSeek) uses a different path structure - this.client = new OpenAI({ - baseURL, - apiKey, - defaultHeaders: headers, - defaultQuery: { "api-version": this.options.azureApiVersion || "2024-05-01-preview" }, - timeout, - }) - } else if (isAzureOpenAi) { - // Azure API shape slightly differs from the core API shape: - // https://github.com/openai/openai-node?tab=readme-ov-file#microsoft-azure-openai - this.client = new AzureOpenAI({ - baseURL, - apiKey, - apiVersion: this.options.azureApiVersion || azureOpenAiDefaultApiVersion, - defaultHeaders: headers, - timeout, - }) + // Create the appropriate handler + if (this.useResponsesApi) { + this.handler = new OpenAIResponsesHandler(options) } else { - this.client = new OpenAI({ - baseURL, - apiKey, - defaultHeaders: headers, - timeout, - }) + this.handler = new OpenAIChatCompletionsHandler(options) } } - override async *createMessage( + /** + * Determines whether to use the Responses API based on URL patterns and configuration + */ + private shouldUseResponsesApi(options: ApiHandlerOptions): boolean { + // Check URL pattern for /v1/responses endpoint + const baseUrl = options.openAiBaseUrl ?? "" + if (baseUrl.includes("/v1/responses") || baseUrl.endsWith("/responses")) { + return true + } + + // Check if it's a GPT-5 model (which should use Responses API) + const modelId = options.openAiModelId ?? "" + if (modelId.startsWith("gpt-5")) { + return true + } + + // Check if it's configured for OpenAI Native (which uses Responses API) + if (options.openAiNativeApiKey && !options.openAiApiKey) { + return true + } + + // Default to Chat Completions API for backward compatibility + return false + } + + async *createMessage( systemPrompt: string, messages: Anthropic.Messages.MessageParam[], metadata?: ApiHandlerCreateMessageMetadata, ): ApiStream { - const { info: modelInfo, reasoning } = this.getModel() - const modelUrl = this.options.openAiBaseUrl ?? "" - const modelId = this.options.openAiModelId ?? "" - const enabledR1Format = this.options.openAiR1FormatEnabled ?? false - const enabledLegacyFormat = this.options.openAiLegacyFormat ?? false - const isAzureAiInference = this._isAzureAiInference(modelUrl) - const deepseekReasoner = modelId.includes("deepseek-reasoner") || enabledR1Format - const ark = modelUrl.includes(".volces.com") - - if (modelId.includes("o1") || modelId.includes("o3") || modelId.includes("o4")) { - yield* this.handleO3FamilyMessage(modelId, systemPrompt, messages) - return - } - - if (this.options.openAiStreamingEnabled ?? true) { - let systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = { - role: "system", - content: systemPrompt, - } - - let convertedMessages - - if (deepseekReasoner) { - convertedMessages = convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) - } else if (ark || enabledLegacyFormat) { - convertedMessages = [systemMessage, ...convertToSimpleMessages(messages)] - } else { - if (modelInfo.supportsPromptCache) { - systemMessage = { - role: "system", - content: [ - { - type: "text", - text: systemPrompt, - // @ts-ignore-next-line - cache_control: { type: "ephemeral" }, - }, - ], - } - } - - convertedMessages = [systemMessage, ...convertToOpenAiMessages(messages)] - - if (modelInfo.supportsPromptCache) { - // Note: the following logic is copied from openrouter: - // Add cache_control to the last two user messages - // (note: this works because we only ever add one user message at a time, but if we added multiple we'd need to mark the user message before the last assistant message) - const lastTwoUserMessages = convertedMessages.filter((msg) => msg.role === "user").slice(-2) - - lastTwoUserMessages.forEach((msg) => { - if (typeof msg.content === "string") { - msg.content = [{ type: "text", text: msg.content }] - } - - if (Array.isArray(msg.content)) { - // NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end. - let lastTextPart = msg.content.filter((part) => part.type === "text").pop() - - if (!lastTextPart) { - lastTextPart = { type: "text", text: "..." } - msg.content.push(lastTextPart) - } - - // @ts-ignore-next-line - lastTextPart["cache_control"] = { type: "ephemeral" } - } - }) - } - } - - const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) - - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { - model: modelId, - temperature: this.options.modelTemperature ?? (deepseekReasoner ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0), - messages: convertedMessages, - stream: true as const, - ...(isGrokXAI ? {} : { stream_options: { include_usage: true } }), - ...(reasoning && reasoning), - } - - // Add max_tokens if needed - this.addMaxTokensIfNeeded(requestOptions, modelInfo) - - let stream - try { - stream = await this.client.chat.completions.create( - requestOptions, - isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, - ) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - - const matcher = new XmlMatcher( - "think", - (chunk) => - ({ - type: chunk.matched ? "reasoning" : "text", - text: chunk.data, - }) as const, - ) - - let lastUsage - - for await (const chunk of stream) { - const delta = chunk.choices[0]?.delta ?? {} - - if (delta.content) { - for (const chunk of matcher.update(delta.content)) { - yield chunk - } - } - - if ("reasoning_content" in delta && delta.reasoning_content) { - yield { - type: "reasoning", - text: (delta.reasoning_content as string | undefined) || "", - } - } - if (chunk.usage) { - lastUsage = chunk.usage - } - } - - for (const chunk of matcher.final()) { - yield chunk - } - - if (lastUsage) { - yield this.processUsageMetrics(lastUsage, modelInfo) - } - } else { - // o1 for instance doesnt support streaming, non-1 temp, or system prompt - const systemMessage: OpenAI.Chat.ChatCompletionUserMessageParam = { - role: "user", - content: systemPrompt, - } - - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { - model: modelId, - messages: deepseekReasoner - ? convertToR1Format([{ role: "user", content: systemPrompt }, ...messages]) - : enabledLegacyFormat - ? [systemMessage, ...convertToSimpleMessages(messages)] - : [systemMessage, ...convertToOpenAiMessages(messages)], - } - - // Add max_tokens if needed - this.addMaxTokensIfNeeded(requestOptions, modelInfo) - - let response - try { - response = await this.client.chat.completions.create( - requestOptions, - this._isAzureAiInference(modelUrl) ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, - ) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - - yield { - type: "text", - text: response.choices[0]?.message.content || "", - } - - yield this.processUsageMetrics(response.usage, modelInfo) - } + yield* this.handler.createMessage(systemPrompt, messages, metadata) } - protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { - return { - type: "usage", - inputTokens: usage?.prompt_tokens || 0, - outputTokens: usage?.completion_tokens || 0, - cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, - cacheReadTokens: usage?.cache_read_input_tokens || undefined, - } - } - - override getModel() { - const id = this.options.openAiModelId ?? "" - const info = this.options.openAiCustomModelInfo ?? openAiModelInfoSaneDefaults - const params = getModelParams({ format: "openai", modelId: id, model: info, settings: this.options }) - return { id, info, ...params } + getModel() { + return this.handler.getModel() } async completePrompt(prompt: string): Promise { - try { - const isAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) - const model = this.getModel() - const modelInfo = model.info - - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { - model: model.id, - messages: [{ role: "user", content: prompt }], - } - - // Add max_tokens if needed - this.addMaxTokensIfNeeded(requestOptions, modelInfo) - - let response - try { - response = await this.client.chat.completions.create( - requestOptions, - isAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, - ) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - - return response.choices[0]?.message.content || "" - } catch (error) { - if (error instanceof Error) { - throw new Error(`${this.providerName} completion error: ${error.message}`) - } - - throw error - } - } - - private async *handleO3FamilyMessage( - modelId: string, - systemPrompt: string, - messages: Anthropic.Messages.MessageParam[], - ): ApiStream { - const modelInfo = this.getModel().info - const methodIsAzureAiInference = this._isAzureAiInference(this.options.openAiBaseUrl) - - if (this.options.openAiStreamingEnabled ?? true) { - const isGrokXAI = this._isGrokXAI(this.options.openAiBaseUrl) - - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { - model: modelId, - messages: [ - { - role: "developer", - content: `Formatting re-enabled\n${systemPrompt}`, - }, - ...convertToOpenAiMessages(messages), - ], - stream: true, - ...(isGrokXAI ? {} : { stream_options: { include_usage: true } }), - reasoning_effort: modelInfo.reasoningEffort as "low" | "medium" | "high" | undefined, - temperature: undefined, - } - - // O3 family models do not support the deprecated max_tokens parameter - // but they do support max_completion_tokens (the modern OpenAI parameter) - // This allows O3 models to limit response length when includeMaxTokens is enabled - this.addMaxTokensIfNeeded(requestOptions, modelInfo) - - let stream - try { - stream = await this.client.chat.completions.create( - requestOptions, - methodIsAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, - ) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - - yield* this.handleStreamResponse(stream) - } else { - const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = { - model: modelId, - messages: [ - { - role: "developer", - content: `Formatting re-enabled\n${systemPrompt}`, - }, - ...convertToOpenAiMessages(messages), - ], - reasoning_effort: modelInfo.reasoningEffort as "low" | "medium" | "high" | undefined, - temperature: undefined, - } - - // O3 family models do not support the deprecated max_tokens parameter - // but they do support max_completion_tokens (the modern OpenAI parameter) - // This allows O3 models to limit response length when includeMaxTokens is enabled - this.addMaxTokensIfNeeded(requestOptions, modelInfo) - - let response - try { - response = await this.client.chat.completions.create( - requestOptions, - methodIsAzureAiInference ? { path: OPENAI_AZURE_AI_INFERENCE_PATH } : {}, - ) - } catch (error) { - throw handleOpenAIError(error, this.providerName) - } - - yield { - type: "text", - text: response.choices[0]?.message.content || "", - } - yield this.processUsageMetrics(response.usage) - } - } - - private async *handleStreamResponse(stream: AsyncIterable): ApiStream { - for await (const chunk of stream) { - const delta = chunk.choices[0]?.delta - if (delta?.content) { - yield { - type: "text", - text: delta.content, - } - } - - if (chunk.usage) { - yield { - type: "usage", - inputTokens: chunk.usage.prompt_tokens || 0, - outputTokens: chunk.usage.completion_tokens || 0, - } - } - } - } - - private _getUrlHost(baseUrl?: string): string { - try { - return new URL(baseUrl ?? "").host - } catch (error) { - return "" - } - } - - private _isGrokXAI(baseUrl?: string): boolean { - const urlHost = this._getUrlHost(baseUrl) - return urlHost.includes("x.ai") - } - - private _isAzureAiInference(baseUrl?: string): boolean { - const urlHost = this._getUrlHost(baseUrl) - return urlHost.endsWith(".services.ai.azure.com") + return this.handler.completePrompt(prompt) } /** - * Adds max_completion_tokens to the request body if needed based on provider configuration - * Note: max_tokens is deprecated in favor of max_completion_tokens as per OpenAI documentation - * O3 family models handle max_tokens separately in handleO3FamilyMessage + * Get information about which API is being used */ - protected addMaxTokensIfNeeded( - requestOptions: - | OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming - | OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, - modelInfo: ModelInfo, - ): void { - // Only add max_completion_tokens if includeMaxTokens is true - if (this.options.includeMaxTokens === true) { - // Use user-configured modelMaxTokens if available, otherwise fall back to model's default maxTokens - // Using max_completion_tokens as max_tokens is deprecated - requestOptions.max_completion_tokens = this.options.modelMaxTokens || modelInfo.maxTokens + getApiType(): "chat-completions" | "responses" { + return this.useResponsesApi ? "responses" : "chat-completions" + } + + /** + * For Responses API, get the last response ID for conversation continuity + */ + getLastResponseId(): string | undefined { + if (this.handler instanceof OpenAIResponsesHandler) { + return this.handler.getLastResponseId() + } + return undefined + } + + /** + * For Responses API, set the response ID for conversation continuity + */ + setResponseId(responseId: string): void { + if (this.handler instanceof OpenAIResponsesHandler) { + this.handler.setResponseId(responseId) } } }