feat(models): add per-model timeout disable to avoid global override for long-running models (e.g., gpt-5-pro)

- Introduce ModelInfo.disableTimeout to opt out of request timeouts on a per-model basis
- Apply in OpenAI-compatible, Ollama, and LM Studio providers (timeout=0 when flag is true)
- Preserve global “API Request Timeout” behavior (0 still disables globally); per-model flag takes precedence for that model
- Motivation: gpt-5-pro often requires longer runtimes; per-model override avoids forcing a global setting that impacts all models
- Add/extend unit tests to validate provider behavior
This commit is contained in:
Hannes Rudolph 2025-10-10 13:27:17 -06:00
parent 24eb6ae984
commit 84f58cd518
5 changed files with 64 additions and 2 deletions

View file

@ -86,6 +86,8 @@ export const modelInfoSchema = z.object({
// Capability flag to indicate whether the model supports temperature parameter
supportsTemperature: z.boolean().optional(),
defaultTemperature: z.number().optional(),
// When true, force-disable request timeouts for this model (providers will set timeout=0)
disableTimeout: z.boolean().optional(),
requiredReasoningBudget: z.boolean().optional(),
supportsReasoningEffort: z
.union([z.boolean(), z.array(z.enum(["disable", "none", "minimal", "low", "medium", "high", "xhigh"]))])

View file

@ -88,4 +88,34 @@ describe("LmStudioHandler timeout configuration", () => {
}),
)
})
it("should force zero timeout when model info disables timeout", () => {
;(getApiRequestTimeout as any).mockReturnValue(600000)
const spy = vitest.spyOn(LmStudioHandler.prototype as any, "getModel").mockReturnValue({
id: "llama2",
info: {
maxTokens: -1,
contextWindow: 128000,
supportsPromptCache: false,
supportsImages: true,
disableTimeout: true,
},
})
const options: ApiHandlerOptions = {
apiModelId: "llama2",
lmStudioModelId: "llama2",
}
new LmStudioHandler(options)
expect(mockOpenAIConstructor).toHaveBeenCalledWith(
expect.objectContaining({
timeout: 0,
}),
)
spy.mockRestore()
})
})

View file

@ -141,4 +141,28 @@ describe("OpenAiHandler timeout configuration", () => {
}),
)
})
it("should force zero timeout when model info disables timeout", () => {
;(getApiRequestTimeout as any).mockReturnValue(600000)
const options: ApiHandlerOptions = {
apiModelId: "gpt-4",
openAiModelId: "gpt-4",
openAiCustomModelInfo: {
maxTokens: -1,
contextWindow: 128000,
supportsPromptCache: false,
supportsImages: true,
disableTimeout: true,
} as any,
}
new OpenAiHandler(options)
expect(mockOpenAIConstructor).toHaveBeenCalledWith(
expect.objectContaining({
timeout: 0, // Forced no timeout via model info
}),
)
})
})

View file

@ -32,7 +32,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
this.client = new OpenAI({
baseURL: (this.options.lmStudioBaseUrl || "http://localhost:1234") + "/v1",
apiKey: apiKey,
timeout: getApiRequestTimeout(),
timeout: this.getModel().info?.disableTimeout === true ? 0 : getApiRequestTimeout(),
})
}

View file

@ -49,7 +49,13 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
...(this.options.openAiHeaders || {}),
}
const timeout = getApiRequestTimeout()
let timeout = getApiRequestTimeout()
try {
const modelInfo = this.getModel().info
if (modelInfo?.disableTimeout === true) {
timeout = 0
}
} catch {}
if (isAzureAiInference) {
// Azure AI Inference Service (e.g., for DeepSeek) uses a different path structure