mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
feat: add custom model context window override for all providers
- Add modelContextWindow field to base provider settings schema - Implement applyModelOverrides method in BaseProvider class - Update all provider implementations to apply context window override - Add comprehensive tests for context window override functionality This allows users to customize the context window size for any provider to work around corporate upload limits or other restrictions. Fixes #8397
This commit is contained in:
parent
702b269a1b
commit
74a1d9c222
12 changed files with 186 additions and 6 deletions
1
.tmp/Roo-Code
Submodule
1
.tmp/Roo-Code
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 86debeef43acbea9bdc1aa4b38d514541e164c91
|
||||
1
.tmp/pr-8396
Submodule
1
.tmp/pr-8396
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit a18538995de0f7d9cfc4a40d31380fb141d5604e
|
||||
|
|
@ -179,6 +179,9 @@ const baseProviderSettingsSchema = z.object({
|
|||
modelMaxTokens: z.number().optional(),
|
||||
modelMaxThinkingTokens: z.number().optional(),
|
||||
|
||||
// Model context window override.
|
||||
modelContextWindow: z.number().optional(),
|
||||
|
||||
// Model verbosity.
|
||||
verbosity: verbosityLevelsSchema.optional(),
|
||||
})
|
||||
|
|
|
|||
126
src/api/providers/__tests__/context-window-override.spec.ts
Normal file
126
src/api/providers/__tests__/context-window-override.spec.ts
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
import { describe, it, expect, beforeEach } from "vitest"
|
||||
import { AnthropicHandler } from "../anthropic"
|
||||
import { OpenRouterHandler } from "../openrouter"
|
||||
import { OpenAiHandler } from "../openai"
|
||||
import { GeminiHandler } from "../gemini"
|
||||
import type { ApiHandlerOptions } from "../../../shared/api"
|
||||
|
||||
describe("Context Window Override", () => {
|
||||
describe("AnthropicHandler", () => {
|
||||
it("should apply modelContextWindow override", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
apiKey: "test-key",
|
||||
apiModelId: "claude-3-5-sonnet-20241022",
|
||||
modelContextWindow: 50000, // Custom context window
|
||||
}
|
||||
|
||||
const handler = new AnthropicHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
expect(model.info.contextWindow).toBe(50000)
|
||||
})
|
||||
|
||||
it("should use default context window when no override is provided", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
apiKey: "test-key",
|
||||
apiModelId: "claude-3-5-sonnet-20241022",
|
||||
}
|
||||
|
||||
const handler = new AnthropicHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
// Should use the default context window for this model
|
||||
expect(model.info.contextWindow).toBe(200000)
|
||||
})
|
||||
})
|
||||
|
||||
describe("OpenRouterHandler", () => {
|
||||
it("should apply modelContextWindow override", async () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
openRouterApiKey: "test-key",
|
||||
openRouterModelId: "anthropic/claude-3.5-sonnet",
|
||||
modelContextWindow: 75000, // Custom context window
|
||||
}
|
||||
|
||||
const handler = new OpenRouterHandler(options)
|
||||
// Mock the models to avoid actual API calls
|
||||
;(handler as any).models = {
|
||||
"anthropic/claude-3.5-sonnet": {
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
supportsPromptCache: true,
|
||||
supportsImages: true,
|
||||
},
|
||||
}
|
||||
|
||||
const model = handler.getModel()
|
||||
expect(model.info.contextWindow).toBe(75000)
|
||||
})
|
||||
})
|
||||
|
||||
describe("OpenAiHandler", () => {
|
||||
it("should apply modelContextWindow override to custom model info", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
openAiApiKey: "test-key",
|
||||
openAiModelId: "gpt-4",
|
||||
openAiCustomModelInfo: {
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
supportsPromptCache: false,
|
||||
supportsImages: true,
|
||||
},
|
||||
modelContextWindow: 60000, // Custom context window
|
||||
}
|
||||
|
||||
const handler = new OpenAiHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
expect(model.info.contextWindow).toBe(60000)
|
||||
})
|
||||
})
|
||||
|
||||
describe("GeminiHandler", () => {
|
||||
it("should apply modelContextWindow override", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
geminiApiKey: "test-key",
|
||||
apiModelId: "gemini-1.5-pro-latest",
|
||||
modelContextWindow: 100000, // Custom context window
|
||||
}
|
||||
|
||||
const handler = new GeminiHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
expect(model.info.contextWindow).toBe(100000)
|
||||
})
|
||||
})
|
||||
|
||||
describe("Edge cases", () => {
|
||||
it("should not apply override when modelContextWindow is 0", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
apiKey: "test-key",
|
||||
apiModelId: "claude-3-5-sonnet-20241022",
|
||||
modelContextWindow: 0, // Zero should not override
|
||||
}
|
||||
|
||||
const handler = new AnthropicHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
// Should use the default context window
|
||||
expect(model.info.contextWindow).toBe(200000)
|
||||
})
|
||||
|
||||
it("should not apply override when modelContextWindow is negative", () => {
|
||||
const options: ApiHandlerOptions = {
|
||||
apiKey: "test-key",
|
||||
apiModelId: "claude-3-5-sonnet-20241022",
|
||||
modelContextWindow: -1000, // Negative should not override
|
||||
}
|
||||
|
||||
const handler = new AnthropicHandler(options)
|
||||
const model = handler.getModel()
|
||||
|
||||
// Should use the default context window
|
||||
expect(model.info.contextWindow).toBe(200000)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -264,6 +264,9 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
|
|||
}
|
||||
}
|
||||
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
info = this.applyModelOverrides(info, this.options)
|
||||
|
||||
const params = getModelParams({
|
||||
format: "anthropic",
|
||||
modelId: id,
|
||||
|
|
|
|||
|
|
@ -140,6 +140,9 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
|
|||
? (this.options.apiModelId as ModelName)
|
||||
: this.defaultProviderModelId
|
||||
|
||||
return { id, info: this.providerModels[id] }
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
const info = this.applyModelOverrides(this.providerModels[id], this.options)
|
||||
|
||||
return { id, info }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
|
|||
import type { ModelInfo } from "@roo-code/types"
|
||||
|
||||
import type { ApiHandler, ApiHandlerCreateMessageMetadata } from "../index"
|
||||
import type { ApiHandlerOptions } from "../../shared/api"
|
||||
import { ApiStream } from "../transform/stream"
|
||||
import { countTokens } from "../../utils/countTokens"
|
||||
|
||||
|
|
@ -18,6 +19,26 @@ export abstract class BaseProvider implements ApiHandler {
|
|||
|
||||
abstract getModel(): { id: string; info: ModelInfo }
|
||||
|
||||
/**
|
||||
* Applies user-configured overrides to model info.
|
||||
* This allows users to customize model parameters like context window size
|
||||
* to work around corporate restrictions or other limitations.
|
||||
*
|
||||
* @param info The original model info
|
||||
* @param options The API handler options containing user overrides
|
||||
* @returns The model info with overrides applied
|
||||
*/
|
||||
protected applyModelOverrides(info: ModelInfo, options: ApiHandlerOptions): ModelInfo {
|
||||
const overriddenInfo = { ...info }
|
||||
|
||||
// Apply context window override if specified
|
||||
if (options.modelContextWindow && options.modelContextWindow > 0) {
|
||||
overriddenInfo.contextWindow = options.modelContextWindow
|
||||
}
|
||||
|
||||
return overriddenInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* Default token counting implementation using tiktoken.
|
||||
* Providers can override this to use their native token counting endpoints.
|
||||
|
|
|
|||
|
|
@ -373,7 +373,7 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
|
|||
maxTokens: modelConfig.maxTokens || (modelConfig.info.maxTokens as number),
|
||||
temperature: modelConfig.temperature ?? (this.options.modelTemperature as number),
|
||||
}
|
||||
|
||||
|
||||
// Check if 1M context is enabled for Claude Sonnet 4
|
||||
// Use parseBaseModelId to handle cross-region inference prefixes
|
||||
const baseModelId = this.parseBaseModelId(modelConfig.id)
|
||||
|
|
@ -922,10 +922,14 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
|
|||
if (this.options.modelMaxTokens && this.options.modelMaxTokens > 0) {
|
||||
model.info.maxTokens = this.options.modelMaxTokens
|
||||
}
|
||||
// Support both awsModelContextWindow (for backward compatibility) and modelContextWindow
|
||||
if (this.options.awsModelContextWindow && this.options.awsModelContextWindow > 0) {
|
||||
model.info.contextWindow = this.options.awsModelContextWindow
|
||||
}
|
||||
|
||||
// Apply general model overrides (including modelContextWindow)
|
||||
model.info = this.applyModelOverrides(model.info, this.options)
|
||||
|
||||
return model
|
||||
}
|
||||
|
||||
|
|
@ -983,6 +987,9 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
|
|||
}
|
||||
}
|
||||
|
||||
// Apply general model overrides (including modelContextWindow) after all specific logic
|
||||
modelConfig.info = this.applyModelOverrides(modelConfig.info, this.options)
|
||||
|
||||
// Get model params including reasoning configuration
|
||||
const params = getModelParams({
|
||||
format: "anthropic",
|
||||
|
|
|
|||
|
|
@ -166,6 +166,10 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
|
|||
const modelId = this.options.apiModelId
|
||||
let id = modelId && modelId in geminiModels ? (modelId as GeminiModelId) : geminiDefaultModelId
|
||||
let info: ModelInfo = geminiModels[id]
|
||||
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
info = this.applyModelOverrides(info, this.options)
|
||||
|
||||
const params = getModelParams({ format: "gemini", modelId: id, model: info, settings: this.options })
|
||||
|
||||
// The `:thinking` suffix indicates that the model is a "Hybrid"
|
||||
|
|
|
|||
|
|
@ -267,7 +267,11 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
|
|||
|
||||
override getModel() {
|
||||
const id = this.options.openAiModelId ?? ""
|
||||
const info = this.options.openAiCustomModelInfo ?? openAiModelInfoSaneDefaults
|
||||
let info = this.options.openAiCustomModelInfo ?? openAiModelInfoSaneDefaults
|
||||
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
info = this.applyModelOverrides(info, this.options)
|
||||
|
||||
const params = getModelParams({ format: "openai", modelId: id, model: info, settings: this.options })
|
||||
return { id, info, ...params }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -232,6 +232,9 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
|
|||
info = this.endpoints[this.options.openRouterSpecificProvider]
|
||||
}
|
||||
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
info = this.applyModelOverrides(info, this.options)
|
||||
|
||||
const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || id === "perplexity/sonar-reasoning"
|
||||
|
||||
const params = getModelParams({
|
||||
|
|
|
|||
|
|
@ -63,9 +63,13 @@ export abstract class RouterProvider extends BaseProvider {
|
|||
override getModel(): { id: string; info: ModelInfo } {
|
||||
const id = this.modelId ?? this.defaultModelId
|
||||
|
||||
return this.models[id]
|
||||
? { id, info: this.models[id] }
|
||||
: { id: this.defaultModelId, info: this.defaultModelInfo }
|
||||
// Get the base model info
|
||||
const baseInfo = this.models[id] ? this.models[id] : this.defaultModelInfo
|
||||
|
||||
// Apply user-configured overrides (e.g., custom context window)
|
||||
const info = this.applyModelOverrides(baseInfo, this.options)
|
||||
|
||||
return { id: this.models[id] ? id : this.defaultModelId, info }
|
||||
}
|
||||
|
||||
protected supportsTemperature(modelId: string): boolean {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue