fix: limit GPT-5 models max output tokens to 10k to prevent context overflow

- Added special handling for GPT-5 models in getModelMaxOutputTokens()
- Limits max output to 10k tokens as recommended in https://github.com/cline/cline/issues/5474#issuecomment-3172109387
- Prevents context window overflow when input approaches 272k token limit
- Added comprehensive tests for GPT-5 token limiting behavior

Fixes #6856

src/shared/__tests__/api.spec.ts

@@ -217,6 +217,121 @@ describe("getModelMaxOutputTokens", () => {
 
 		expect(getModelMaxOutputTokens({ modelId: "test", model, settings })).toBe(16_384)
 	})
+
+	describe("GPT-5 models token limit", () => {
+		test("should limit GPT-5 models to 10k max output tokens", () => {
+			const gpt5Model: ModelInfo = {
+				contextWindow: 400_000,
+				maxTokens: 128_000,
+				supportsPromptCache: true,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-5-2025-08-07",
+				model: gpt5Model,
+				settings: {},
+				format: "openai",
+			})
+
+			expect(result).toBe(10_000)
+		})
+
+		test("should limit GPT-5-mini models to 10k max output tokens", () => {
+			const gpt5MiniModel: ModelInfo = {
+				contextWindow: 400_000,
+				maxTokens: 128_000,
+				supportsPromptCache: true,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-5-mini-2025-08-07",
+				model: gpt5MiniModel,
+				settings: {},
+				format: "openai",
+			})
+
+			expect(result).toBe(10_000)
+		})
+
+		test("should limit GPT-5-nano models to 10k max output tokens", () => {
+			const gpt5NanoModel: ModelInfo = {
+				contextWindow: 400_000,
+				maxTokens: 128_000,
+				supportsPromptCache: true,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-5-nano-2025-08-07",
+				model: gpt5NanoModel,
+				settings: {},
+				format: "openai",
+			})
+
+			expect(result).toBe(10_000)
+		})
+
+		test("should respect user override for GPT-5 models but cap at 10k", () => {
+			const gpt5Model: ModelInfo = {
+				contextWindow: 400_000,
+				maxTokens: 128_000,
+				supportsPromptCache: true,
+			}
+
+			// User tries to set 15k, should be capped at 10k
+			const settings: ProviderSettings = {
+				modelMaxTokens: 15_000,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-5-2025-08-07",
+				model: gpt5Model,
+				settings,
+				format: "openai",
+			})
+
+			expect(result).toBe(10_000)
+		})
+
+		test("should allow user to set lower than 10k for GPT-5 models", () => {
+			const gpt5Model: ModelInfo = {
+				contextWindow: 400_000,
+				maxTokens: 128_000,
+				supportsPromptCache: true,
+			}
+
+			// User sets 5k, should be respected
+			const settings: ProviderSettings = {
+				modelMaxTokens: 5_000,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-5-2025-08-07",
+				model: gpt5Model,
+				settings,
+				format: "openai",
+			})
+
+			expect(result).toBe(5_000)
+		})
+
+		test("should not affect non-GPT-5 models", () => {
+			const gpt4Model: ModelInfo = {
+				contextWindow: 128_000,
+				maxTokens: 16_384,
+				supportsPromptCache: true,
+			}
+
+			const result = getModelMaxOutputTokens({
+				modelId: "gpt-4o",
+				model: gpt4Model,
+				settings: {},
+				format: "openai",
+			})
+
+			// Should use model's maxTokens since it's within 20% of context window
+			expect(result).toBe(16_384)
+		})
+	})
 })
 
 describe("shouldUseReasoningBudget", () => {

src/shared/api.ts

@@ -87,6 +87,17 @@ export const getModelMaxOutputTokens = ({
 		return settings.claudeCodeMaxOutputTokens || CLAUDE_CODE_DEFAULT_MAX_OUTPUT_TOKENS
 	}
 
+	// Special handling for GPT-5 models to prevent context window overflow
+	// Limit max output to 10k tokens as per https://github.com/cline/cline/issues/5474#issuecomment-3172109387
+	if (modelId.startsWith("gpt-5")) {
+		// Allow user override via settings, but cap at 10k
+		const userMaxTokens = settings?.modelMaxTokens
+		if (userMaxTokens) {
+			return Math.min(userMaxTokens, 10000)
+		}
+		return 10000
+	}
+
 	if (shouldUseReasoningBudget({ model, settings })) {
 		return settings?.modelMaxTokens || DEFAULT_HYBRID_REASONING_MODEL_MAX_TOKENS
 	}