mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
fix: prevent OpenRouter context overflow by capping max_completion_tokens
- Fix issue where OpenRouter models like moonshotai/kimi-k2 fail with context overflow - Cap max_completion_tokens to 20% of context window when it equals full context window - Refine GPT-5 model detection to prevent false positives with OpenRouter models - Add comprehensive test coverage for edge cases Fixes #5658
This commit is contained in:
parent
195f4eb245
commit
56f619b51d
4 changed files with 178 additions and 3 deletions
|
|
@ -382,5 +382,80 @@ describe("OpenRouter API", () => {
|
|||
expect(textResult.maxTokens).toBe(64000)
|
||||
expect(imageResult.maxTokens).toBe(64000)
|
||||
})
|
||||
|
||||
it("handles context overflow issue - caps max_completion_tokens when it equals context window", () => {
|
||||
const mockModel = {
|
||||
name: "Kimi K2",
|
||||
description: "Test model with context overflow issue",
|
||||
context_length: 131072,
|
||||
max_completion_tokens: 131072, // This equals context window, causing the issue
|
||||
pricing: {
|
||||
prompt: "0.000003",
|
||||
completion: "0.000015",
|
||||
},
|
||||
}
|
||||
|
||||
const result = parseOpenRouterModel({
|
||||
id: "moonshotai/kimi-k2",
|
||||
model: mockModel,
|
||||
inputModality: ["text"],
|
||||
outputModality: ["text"],
|
||||
maxTokens: 131072, // This would cause context overflow
|
||||
})
|
||||
|
||||
// Should cap to 20% of context window instead of using full context window
|
||||
expect(result.maxTokens).toBe(Math.ceil(131072 * 0.2)) // 26215
|
||||
expect(result.contextWindow).toBe(131072)
|
||||
})
|
||||
|
||||
it("uses provided max_completion_tokens when it's reasonable (less than context window)", () => {
|
||||
const mockModel = {
|
||||
name: "Reasonable Model",
|
||||
description: "Test model with reasonable max_completion_tokens",
|
||||
context_length: 131072,
|
||||
max_completion_tokens: 65536, // Half of context window - reasonable
|
||||
pricing: {
|
||||
prompt: "0.000003",
|
||||
completion: "0.000015",
|
||||
},
|
||||
}
|
||||
|
||||
const result = parseOpenRouterModel({
|
||||
id: "test/reasonable-model",
|
||||
model: mockModel,
|
||||
inputModality: ["text"],
|
||||
outputModality: ["text"],
|
||||
maxTokens: 65536,
|
||||
})
|
||||
|
||||
// Should use the provided maxTokens since it's reasonable
|
||||
expect(result.maxTokens).toBe(65536)
|
||||
expect(result.contextWindow).toBe(131072)
|
||||
})
|
||||
|
||||
it("falls back to 20% of context window when max_completion_tokens is not provided", () => {
|
||||
const mockModel = {
|
||||
name: "No Max Tokens Model",
|
||||
description: "Test model without max_completion_tokens",
|
||||
context_length: 100000,
|
||||
max_completion_tokens: null,
|
||||
pricing: {
|
||||
prompt: "0.000003",
|
||||
completion: "0.000015",
|
||||
},
|
||||
}
|
||||
|
||||
const result = parseOpenRouterModel({
|
||||
id: "test/no-max-tokens",
|
||||
model: mockModel,
|
||||
inputModality: ["text"],
|
||||
outputModality: ["text"],
|
||||
maxTokens: null,
|
||||
})
|
||||
|
||||
// Should fall back to 20% of context window
|
||||
expect(result.maxTokens).toBe(Math.ceil(100000 * 0.2)) // 20000
|
||||
expect(result.contextWindow).toBe(100000)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -206,8 +206,20 @@ export const parseOpenRouterModel = ({
|
|||
|
||||
const supportsPromptCache = typeof cacheReadsPrice !== "undefined" // some models support caching but don't charge a cacheWritesPrice, e.g. GPT-5
|
||||
|
||||
// Calculate safe max output tokens
|
||||
// If maxTokens from OpenRouter equals or exceeds the context window, use 20% of context window instead
|
||||
// This prevents the "max_tokens equals context window" issue that causes API failures
|
||||
let safeMaxTokens: number
|
||||
if (maxTokens && maxTokens < model.context_length) {
|
||||
// Use the provided max_completion_tokens if it's reasonable (less than context window)
|
||||
safeMaxTokens = maxTokens
|
||||
} else {
|
||||
// Fall back to 20% of context window for safety
|
||||
safeMaxTokens = Math.ceil(model.context_length * 0.2)
|
||||
}
|
||||
|
||||
const modelInfo: ModelInfo = {
|
||||
maxTokens: maxTokens || Math.ceil(model.context_length * 0.2),
|
||||
maxTokens: safeMaxTokens,
|
||||
contextWindow: model.context_length,
|
||||
supportsImages: inputModality?.includes("image") ?? false,
|
||||
supportsPromptCache,
|
||||
|
|
|
|||
|
|
@ -283,6 +283,90 @@ describe("getModelMaxOutputTokens", () => {
|
|||
})
|
||||
})
|
||||
|
||||
test("should not treat OpenRouter GPT-5 models as native GPT-5 models", () => {
|
||||
const model: ModelInfo = {
|
||||
contextWindow: 200_000,
|
||||
supportsPromptCache: false,
|
||||
maxTokens: 128_000, // 64% of context window, should be capped for OpenRouter models
|
||||
}
|
||||
|
||||
const settings: ProviderSettings = {
|
||||
apiProvider: "openrouter",
|
||||
}
|
||||
|
||||
// Test OpenRouter GPT-5 model IDs that should be capped (non-OpenAI providers)
|
||||
const openRouterGpt5ModelIds = [
|
||||
"openrouter/gpt-5",
|
||||
"some-provider/gpt-5-preview",
|
||||
"anthropic/gpt-5", // hypothetical
|
||||
"moonshotai/gpt-5-like", // hypothetical
|
||||
]
|
||||
|
||||
openRouterGpt5ModelIds.forEach((modelId) => {
|
||||
const result = getModelMaxOutputTokens({
|
||||
modelId,
|
||||
model,
|
||||
settings,
|
||||
format: "openrouter",
|
||||
})
|
||||
// Should be capped to 20% of context window: 200_000 * 0.2 = 40_000
|
||||
// NOT the full 128_000 like native GPT-5 models
|
||||
expect(result).toBe(40_000)
|
||||
})
|
||||
})
|
||||
|
||||
test("should still bypass 20% cap for native GPT-5 models (without slash)", () => {
|
||||
const model: ModelInfo = {
|
||||
contextWindow: 200_000,
|
||||
supportsPromptCache: false,
|
||||
maxTokens: 128_000, // 64% of context window
|
||||
}
|
||||
|
||||
const settings: ProviderSettings = {
|
||||
apiProvider: "openai",
|
||||
}
|
||||
|
||||
// Test native GPT-5 model IDs (without slash)
|
||||
const nativeGpt5ModelIds = ["gpt-5", "gpt-5-turbo", "GPT-5", "gpt-5-32k"]
|
||||
|
||||
nativeGpt5ModelIds.forEach((modelId) => {
|
||||
const result = getModelMaxOutputTokens({
|
||||
modelId,
|
||||
model,
|
||||
settings,
|
||||
format: "openai",
|
||||
})
|
||||
// Should use full 128k tokens, not capped to 20% (40k)
|
||||
expect(result).toBe(128_000)
|
||||
})
|
||||
})
|
||||
|
||||
test("should still bypass 20% cap for OpenAI GPT-5 models through OpenRouter", () => {
|
||||
const model: ModelInfo = {
|
||||
contextWindow: 200_000,
|
||||
supportsPromptCache: false,
|
||||
maxTokens: 128_000, // 64% of context window
|
||||
}
|
||||
|
||||
const settings: ProviderSettings = {
|
||||
apiProvider: "openrouter",
|
||||
}
|
||||
|
||||
// Test OpenAI GPT-5 model IDs through OpenRouter (should still bypass cap)
|
||||
const openaiGpt5ModelIds = ["openai/gpt-5", "openai/gpt-5-turbo", "openai/gpt-5-preview"]
|
||||
|
||||
openaiGpt5ModelIds.forEach((modelId) => {
|
||||
const result = getModelMaxOutputTokens({
|
||||
modelId,
|
||||
model,
|
||||
settings,
|
||||
format: "openrouter",
|
||||
})
|
||||
// Should use full 128k tokens, not capped to 20% (40k)
|
||||
expect(result).toBe(128_000)
|
||||
})
|
||||
})
|
||||
|
||||
test("should return modelMaxTokens from settings when reasoning budget is required", () => {
|
||||
const model: ModelInfo = {
|
||||
contextWindow: 200_000,
|
||||
|
|
|
|||
|
|
@ -122,14 +122,18 @@ export const getModelMaxOutputTokens = ({
|
|||
// Exception: GPT-5 models should use their exact configured max output tokens
|
||||
if (model.maxTokens) {
|
||||
// Check if this is a GPT-5 model (case-insensitive)
|
||||
const isGpt5Model = modelId.toLowerCase().includes("gpt-5")
|
||||
// Make sure we don't incorrectly identify OpenRouter models as GPT-5
|
||||
// OpenRouter models typically have format "provider/model" but native OpenAI models can be "openai/gpt-5"
|
||||
const isGpt5Model =
|
||||
modelId.toLowerCase().includes("gpt-5") && (format !== "openrouter" || modelId.startsWith("openai/"))
|
||||
|
||||
// GPT-5 models bypass the 20% cap and use their full configured max tokens
|
||||
if (isGpt5Model) {
|
||||
return model.maxTokens
|
||||
}
|
||||
|
||||
// All other models are clamped to 20% of context window
|
||||
// All other models (including OpenRouter models) are clamped to 20% of context window
|
||||
// This prevents context overflow issues where max_tokens equals the full context window
|
||||
return Math.min(model.maxTokens, Math.ceil(model.contextWindow * 0.2))
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue