diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index 207c60a524..ee52a69ce4 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -144,6 +144,7 @@ const openAiSchema = baseProviderSettingsSchema.extend({ openAiStreamingEnabled: z.boolean().optional(), openAiHostHeader: z.string().optional(), // Keep temporarily for backward compatibility during migration. openAiHeaders: z.record(z.string(), z.string()).optional(), + openAiRequestTimeout: z.number().optional(), // Timeout in milliseconds for OpenAI API requests }) const ollamaSchema = baseProviderSettingsSchema.extend({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e7bb79b64..4f3eb79287 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -685,7 +685,7 @@ importers: version: 12.0.0 openai: specifier: ^5.0.0 - version: 5.5.1(ws@8.18.2)(zod@3.25.61) + version: 5.5.1(ws@8.18.3)(zod@3.25.61) os-name: specifier: ^6.0.0 version: 6.1.0 @@ -758,6 +758,9 @@ importers: turndown: specifier: ^7.2.0 version: 7.2.0 + undici: + specifier: '>=5.29.0' + version: 6.21.3 uuid: specifier: ^11.1.0 version: 11.1.0 @@ -17520,9 +17523,9 @@ snapshots: is-inside-container: 1.0.0 is-wsl: 3.1.0 - openai@5.5.1(ws@8.18.2)(zod@3.25.61): + openai@5.5.1(ws@8.18.3)(zod@3.25.61): optionalDependencies: - ws: 8.18.2 + ws: 8.18.3 zod: 3.25.61 option@0.2.4: {} diff --git a/src/api/providers/__tests__/openai-timeout-integration.spec.ts b/src/api/providers/__tests__/openai-timeout-integration.spec.ts new file mode 100644 index 0000000000..7da1c3fdd9 --- /dev/null +++ b/src/api/providers/__tests__/openai-timeout-integration.spec.ts @@ -0,0 +1,200 @@ +// npx vitest run api/providers/__tests__/openai-timeout-integration.spec.ts + +import { describe, it, expect, vi, beforeEach } from "vitest" +import { OpenAiHandler } from "../openai" +import { BaseOpenAiCompatibleProvider } from "../base-openai-compatible-provider" +import { ApiHandlerOptions } from "../../../shared/api" +import type { ModelInfo } from "@roo-code/types" + +// Mock OpenAI module +const mockCreate = vi.fn() +const mockOpenAIConstructor = vi.fn() + +vi.mock("openai", () => ({ + default: class MockOpenAI { + constructor(config: any) { + mockOpenAIConstructor(config) + return { + chat: { + completions: { + create: mockCreate, + }, + }, + } + } + }, + AzureOpenAI: class MockAzureOpenAI { + constructor(config: any) { + mockOpenAIConstructor(config) + return { + chat: { + completions: { + create: mockCreate, + }, + }, + } + } + }, +})) + +// Test provider implementation +class TestProvider extends BaseOpenAiCompatibleProvider<"test-model"> { + constructor(options: ApiHandlerOptions) { + super({ + ...options, + providerName: "Test Provider", + baseURL: "https://api.test.com", + defaultProviderModelId: "test-model", + providerModels: { + "test-model": { + contextWindow: 128000, + maxTokens: 4096, + supportsImages: false, + supportsPromptCache: false, + } as ModelInfo, + }, + }) + } +} + +describe("OpenAI timeout integration", () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe("OpenAiHandler", () => { + it("should pass timeout configuration to OpenAI client", () => { + const options: ApiHandlerOptions = { + openAiApiKey: "test-key", + openAiModelId: "gpt-4", + openAiRequestTimeout: 600000, // 10 minutes + } + + new OpenAiHandler(options) + + // Check that OpenAI constructor was called with fetch option + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + expect(typeof constructorCall.fetch).toBe("function") + }) + + it("should work without timeout configuration", () => { + const options: ApiHandlerOptions = { + openAiApiKey: "test-key", + openAiModelId: "gpt-4", + } + + new OpenAiHandler(options) + + // Should still have fetch function even without explicit timeout + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + }) + + it("should handle Azure OpenAI configuration with timeout", () => { + const options: ApiHandlerOptions = { + openAiApiKey: "test-key", + openAiModelId: "gpt-4", + openAiBaseUrl: "https://test.openai.azure.com", + azureApiVersion: "2024-05-01-preview", + openAiRequestTimeout: 1200000, // 20 minutes + } + + new OpenAiHandler(options) + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + expect(constructorCall.baseURL).toBe("https://test.openai.azure.com") + }) + + it("should handle Azure AI Inference Service with timeout", () => { + const options: ApiHandlerOptions = { + openAiApiKey: "test-key", + openAiModelId: "deepseek-v3", + openAiBaseUrl: "https://test.services.ai.azure.com", + openAiRequestTimeout: 1800000, // 30 minutes + } + + new OpenAiHandler(options) + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + expect(constructorCall.baseURL).toBe("https://test.services.ai.azure.com") + }) + }) + + describe("BaseOpenAiCompatibleProvider", () => { + it("should use timeout configuration in derived providers", () => { + const options: ApiHandlerOptions = { + apiKey: "test-key", + openAiRequestTimeout: 900000, // 15 minutes + } + + new TestProvider(options) + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + expect(constructorCall.apiKey).toBe("test-key") + expect(constructorCall.baseURL).toBe("https://api.test.com") + }) + + it("should use default timeout when not specified", () => { + const options: ApiHandlerOptions = { + apiKey: "test-key", + } + + new TestProvider(options) + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + }) + + it("should handle zero timeout value", () => { + const options: ApiHandlerOptions = { + apiKey: "test-key", + openAiRequestTimeout: 0, + } + + new TestProvider(options) + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + }) + }) + + describe("timeout behavior", () => { + it("should allow very large timeouts for slow local models", () => { + const options: ApiHandlerOptions = { + openAiApiKey: "test-key", + openAiModelId: "local-llama", + openAiRequestTimeout: 7200000, // 2 hours + } + + expect(() => new OpenAiHandler(options)).not.toThrow() + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + }) + + it("should handle negative timeout values gracefully", () => { + const options: ApiHandlerOptions = { + apiKey: "test-key", + openAiRequestTimeout: -5000, + } + + expect(() => new TestProvider(options)).not.toThrow() + + expect(mockOpenAIConstructor).toHaveBeenCalled() + const constructorCall = mockOpenAIConstructor.mock.calls[0][0] + expect(constructorCall).toHaveProperty("fetch") + }) + }) +}) diff --git a/src/api/providers/__tests__/openai.spec.ts b/src/api/providers/__tests__/openai.spec.ts index b4b5f29204..0f9ac8ad22 100644 --- a/src/api/providers/__tests__/openai.spec.ts +++ b/src/api/providers/__tests__/openai.spec.ts @@ -107,6 +107,7 @@ describe("OpenAiHandler", () => { "X-Title": "Roo Code", "User-Agent": `RooCode/${Package.version}`, }, + fetch: expect.any(Function), }) }) }) diff --git a/src/api/providers/base-openai-compatible-provider.ts b/src/api/providers/base-openai-compatible-provider.ts index f196b5f309..88c444b9bf 100644 --- a/src/api/providers/base-openai-compatible-provider.ts +++ b/src/api/providers/base-openai-compatible-provider.ts @@ -10,6 +10,7 @@ import { convertToOpenAiMessages } from "../transform/openai-format" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" import { DEFAULT_HEADERS } from "./constants" import { BaseProvider } from "./base-provider" +import { createCustomFetch, DEFAULT_OPENAI_REQUEST_TIMEOUT } from "./utils/custom-fetch" type BaseOpenAiCompatibleProviderOptions = ApiHandlerOptions & { providerName: string @@ -55,10 +56,14 @@ export abstract class BaseOpenAiCompatibleProvider throw new Error("API key is required") } + // Create custom fetch with timeout if configured + const customFetch = createCustomFetch(this.options.openAiRequestTimeout || DEFAULT_OPENAI_REQUEST_TIMEOUT) + this.client = new OpenAI({ baseURL, apiKey: this.options.apiKey, defaultHeaders: DEFAULT_HEADERS, + fetch: customFetch, }) } diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index f5e4e4c985..efeba8814f 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -23,6 +23,7 @@ import { getModelParams } from "../transform/model-params" import { DEFAULT_HEADERS } from "./constants" import { BaseProvider } from "./base-provider" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" +import { createCustomFetch, DEFAULT_OPENAI_REQUEST_TIMEOUT } from "./utils/custom-fetch" // TODO: Rename this to OpenAICompatibleHandler. Also, I think the // `OpenAINativeHandler` can subclass from this, since it's obviously @@ -46,6 +47,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl ...(this.options.openAiHeaders || {}), } + // Create custom fetch with timeout if configured + const customFetch = createCustomFetch(this.options.openAiRequestTimeout || DEFAULT_OPENAI_REQUEST_TIMEOUT) + if (isAzureAiInference) { // Azure AI Inference Service (e.g., for DeepSeek) uses a different path structure this.client = new OpenAI({ @@ -53,6 +57,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl apiKey, defaultHeaders: headers, defaultQuery: { "api-version": this.options.azureApiVersion || "2024-05-01-preview" }, + fetch: customFetch, }) } else if (isAzureOpenAi) { // Azure API shape slightly differs from the core API shape: @@ -62,12 +67,14 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl apiKey, apiVersion: this.options.azureApiVersion || azureOpenAiDefaultApiVersion, defaultHeaders: headers, + fetch: customFetch, }) } else { this.client = new OpenAI({ baseURL, apiKey, defaultHeaders: headers, + fetch: customFetch, }) } } diff --git a/src/api/providers/utils/custom-fetch.ts b/src/api/providers/utils/custom-fetch.ts new file mode 100644 index 0000000000..e99b71f662 --- /dev/null +++ b/src/api/providers/utils/custom-fetch.ts @@ -0,0 +1,39 @@ +import { fetch as undiciFetch, type RequestInit as UndiciRequestInit } from "undici" + +/** + * Creates a custom fetch function with configurable timeout for OpenAI providers. + * This addresses the issue where undici's default bodyTimeout of 5 minutes + * causes problems with slow local models during prompt processing. + * + * @param timeout - Timeout in milliseconds. If not provided, uses default behavior. + * @returns A fetch function compatible with the Fetch API + */ +export function createCustomFetch(timeout?: number): typeof fetch { + if (!timeout) { + // If no timeout is specified, return the standard fetch + return fetch + } + + return function customFetch(input: RequestInfo | URL, init?: RequestInit): Promise { + // Convert standard fetch parameters to Undici format with extended timeout options + const undiciOptions: UndiciRequestInit = { + ...init, + // bodyTimeout controls how long to wait for the response body + // This is what causes the 5-minute timeout issue with slow models + bodyTimeout: timeout, + // headersTimeout controls how long to wait for response headers + headersTimeout: timeout, + } as UndiciRequestInit + + // Use undici's fetch with extended timeout options + // Type assertions handle compatibility between fetch and undici types + return undiciFetch(input as any, undiciOptions) as any + } +} + +/** + * Default timeout for OpenAI API requests (30 minutes in milliseconds). + * This provides a reasonable default for slow local models while still + * preventing indefinite hangs. + */ +export const DEFAULT_OPENAI_REQUEST_TIMEOUT = 30 * 60 * 1000 // 30 minutes diff --git a/src/package.json b/src/package.json index 91eb40dde6..bcf20fd4dd 100644 --- a/src/package.json +++ b/src/package.json @@ -476,6 +476,7 @@ "tmp": "^0.2.3", "tree-sitter-wasms": "^0.1.12", "turndown": "^7.2.0", + "undici": "^5.29.0", "uuid": "^11.1.0", "vscode-material-icons": "^0.1.1", "web-tree-sitter": "^0.25.6", diff --git a/webview-ui/src/components/settings/providers/OpenAICompatible.tsx b/webview-ui/src/components/settings/providers/OpenAICompatible.tsx index 736b0253c4..6868086814 100644 --- a/webview-ui/src/components/settings/providers/OpenAICompatible.tsx +++ b/webview-ui/src/components/settings/providers/OpenAICompatible.tsx @@ -177,6 +177,24 @@ export const OpenAICompatible = ({ {t("settings:includeMaxOutputTokensDescription")} +
+ { + const value = (e.target as HTMLInputElement).value + const parsed = parseInt(value) + return isNaN(parsed) || parsed === 0 ? undefined : parsed + })} + placeholder="1800000" + className="w-full"> + + +
+ {t("settings:providers.requestTimeoutDescription")} +
+
diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 46c15556c8..6e6405f6fa 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -227,6 +227,8 @@ "searchProviderPlaceholder": "Search providers", "noProviderMatchFound": "No providers found", "noMatchFound": "No matching profiles found", + "requestTimeout": "Request Timeout (ms)", + "requestTimeoutDescription": "Timeout in milliseconds for API requests. Useful for slow local models that take time to process prompts. Default: 1800000 (30 minutes)", "vscodeLmDescription": " The VS Code Language Model API allows you to run models provided by other VS Code extensions (including but not limited to GitHub Copilot). The easiest way to get started is to install the Copilot and Copilot Chat extensions from the VS Code Marketplace.", "awsCustomArnUse": "Enter a valid Amazon Bedrock ARN for the model you want to use. Format examples:", "awsCustomArnDesc": "Make sure the region in the ARN matches your selected AWS Region above.",