diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index ad58a1cf6b..8c5a1795b1 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -31,7 +31,7 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { let stream: AnthropicStream const cacheControl: CacheControlEphemeral = { type: "ephemeral" } let { id: modelId, info: modelInfo } = this.getModel() - const maxTokens = modelInfo.maxTokens || 8192 + const maxTokens = this.options.modelMaxTokens || modelInfo.maxTokens || 8192 let temperature = this.options.modelTemperature ?? ANTHROPIC_DEFAULT_TEMPERATURE let thinking: BetaThinkingConfigParam | undefined = undefined @@ -41,7 +41,15 @@ export class AnthropicHandler implements ApiHandler, SingleCompletionHandler { // `claude-3-7-sonnet-20250219` model with a thinking budget. // We can handle this more elegantly in the future. modelId = "claude-3-7-sonnet-20250219" - const budgetTokens = this.options.anthropicThinking ?? Math.max(maxTokens * 0.8, 1024) + + // Clamp the thinking budget to be at most 80% of max tokens and at + // least 1024 tokens. + const maxBudgetTokens = Math.floor(maxTokens * 0.8) + const budgetTokens = Math.max( + Math.min(this.options.anthropicThinking ?? maxBudgetTokens, maxBudgetTokens), + 1024, + ) + thinking = { type: "enabled", budget_tokens: budgetTokens } temperature = 1.0 } diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 0a9488e816..69bcb0074c 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -108,12 +108,19 @@ export class OpenRouterHandler implements ApiHandler, SingleCompletionHandler { topP = 0.95 } + const maxTokens = this.options.modelMaxTokens || modelInfo.maxTokens let temperature = this.options.modelTemperature ?? defaultTemperature let thinking: BetaThinkingConfigParam | undefined = undefined if (modelInfo.thinking) { - const maxTokens = modelInfo.maxTokens || 8192 - const budgetTokens = this.options.anthropicThinking ?? Math.max(maxTokens * 0.8, 1024) + // Clamp the thinking budget to be at most 80% of max tokens and at + // least 1024 tokens. + const maxBudgetTokens = Math.floor((maxTokens || 8192) * 0.8) + const budgetTokens = Math.max( + Math.min(this.options.anthropicThinking ?? maxBudgetTokens, maxBudgetTokens), + 1024, + ) + thinking = { type: "enabled", budget_tokens: budgetTokens } temperature = 1.0 } @@ -271,7 +278,7 @@ export async function getOpenRouterModels() { modelInfo.supportsPromptCache = true modelInfo.cacheWritesPrice = 3.75 modelInfo.cacheReadsPrice = 0.3 - modelInfo.maxTokens = 16384 + modelInfo.maxTokens = 64_000 break case rawModel.id.startsWith("anthropic/claude-3.5-sonnet-20240620"): modelInfo.supportsPromptCache = true diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 073bd10911..fb123e0584 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -87,6 +87,7 @@ export type ClineOptions = { export class Cline { readonly taskId: string + readonly apiConfiguration: ApiConfiguration api: ApiHandler private terminalManager: TerminalManager private urlContentFetcher: UrlContentFetcher @@ -148,6 +149,7 @@ export class Cline { } this.taskId = crypto.randomUUID() + this.apiConfiguration = apiConfiguration this.api = buildApiHandler(apiConfiguration) this.terminalManager = new TerminalManager() this.urlContentFetcher = new UrlContentFetcher(provider.context) @@ -961,13 +963,21 @@ export class Cline { cacheWrites = 0, cacheReads = 0, }: ClineApiReqInfo = JSON.parse(previousRequest) + const totalTokens = tokensIn + tokensOut + cacheWrites + cacheReads - const trimmedMessages = truncateConversationIfNeeded( - this.apiConversationHistory, + const modelInfo = this.api.getModel().info + const maxTokens = modelInfo.thinking + ? this.apiConfiguration.modelMaxTokens || modelInfo.maxTokens + : modelInfo.maxTokens + const contextWindow = modelInfo.contextWindow + + const trimmedMessages = truncateConversationIfNeeded({ + messages: this.apiConversationHistory, totalTokens, - this.api.getModel().info, - ) + maxTokens, + contextWindow, + }) if (trimmedMessages !== this.apiConversationHistory) { await this.overwriteApiConversationHistory(trimmedMessages) diff --git a/src/core/sliding-window/__tests__/sliding-window.test.ts b/src/core/sliding-window/__tests__/sliding-window.test.ts index 3dcf9e5fd2..cb897aa8cb 100644 --- a/src/core/sliding-window/__tests__/sliding-window.test.ts +++ b/src/core/sliding-window/__tests__/sliding-window.test.ts @@ -119,11 +119,21 @@ describe("getMaxTokens", () => { // Max tokens = 100000 - 50000 = 50000 // Below max tokens - no truncation - const result1 = truncateConversationIfNeeded(messages, 49999, modelInfo) + const result1 = truncateConversationIfNeeded({ + messages, + totalTokens: 49999, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result1).toEqual(messages) // Above max tokens - truncate - const result2 = truncateConversationIfNeeded(messages, 50001, modelInfo) + const result2 = truncateConversationIfNeeded({ + messages, + totalTokens: 50001, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result2).not.toEqual(messages) expect(result2.length).toBe(3) // Truncated with 0.5 fraction }) @@ -133,11 +143,21 @@ describe("getMaxTokens", () => { // Max tokens = 100000 - (100000 * 0.2) = 80000 // Below max tokens - no truncation - const result1 = truncateConversationIfNeeded(messages, 79999, modelInfo) + const result1 = truncateConversationIfNeeded({ + messages, + totalTokens: 79999, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result1).toEqual(messages) // Above max tokens - truncate - const result2 = truncateConversationIfNeeded(messages, 80001, modelInfo) + const result2 = truncateConversationIfNeeded({ + messages, + totalTokens: 80001, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result2).not.toEqual(messages) expect(result2.length).toBe(3) // Truncated with 0.5 fraction }) @@ -147,11 +167,21 @@ describe("getMaxTokens", () => { // Max tokens = 50000 - 10000 = 40000 // Below max tokens - no truncation - const result1 = truncateConversationIfNeeded(messages, 39999, modelInfo) + const result1 = truncateConversationIfNeeded({ + messages, + totalTokens: 39999, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result1).toEqual(messages) // Above max tokens - truncate - const result2 = truncateConversationIfNeeded(messages, 40001, modelInfo) + const result2 = truncateConversationIfNeeded({ + messages, + totalTokens: 40001, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result2).not.toEqual(messages) expect(result2.length).toBe(3) // Truncated with 0.5 fraction }) @@ -161,11 +191,21 @@ describe("getMaxTokens", () => { // Max tokens = 200000 - 30000 = 170000 // Below max tokens - no truncation - const result1 = truncateConversationIfNeeded(messages, 169999, modelInfo) + const result1 = truncateConversationIfNeeded({ + messages, + totalTokens: 169999, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result1).toEqual(messages) // Above max tokens - truncate - const result2 = truncateConversationIfNeeded(messages, 170001, modelInfo) + const result2 = truncateConversationIfNeeded({ + messages, + totalTokens: 170001, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result2).not.toEqual(messages) expect(result2.length).toBe(3) // Truncated with 0.5 fraction }) @@ -194,7 +234,12 @@ describe("truncateConversationIfNeeded", () => { const maxTokens = 100000 - 30000 // 70000 const totalTokens = 69999 // Below threshold - const result = truncateConversationIfNeeded(messages, totalTokens, modelInfo) + const result = truncateConversationIfNeeded({ + messages, + totalTokens, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result).toEqual(messages) // No truncation occurs }) @@ -207,7 +252,12 @@ describe("truncateConversationIfNeeded", () => { // With 4 messages after the first, 0.5 fraction means remove 2 messages const expectedResult = [messages[0], messages[3], messages[4]] - const result = truncateConversationIfNeeded(messages, totalTokens, modelInfo) + const result = truncateConversationIfNeeded({ + messages, + totalTokens, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) expect(result).toEqual(expectedResult) }) @@ -218,14 +268,38 @@ describe("truncateConversationIfNeeded", () => { // Test below threshold const belowThreshold = 69999 - expect(truncateConversationIfNeeded(messages, belowThreshold, modelInfo1)).toEqual( - truncateConversationIfNeeded(messages, belowThreshold, modelInfo2), + expect( + truncateConversationIfNeeded({ + messages, + totalTokens: belowThreshold, + contextWindow: modelInfo1.contextWindow, + maxTokens: modelInfo1.maxTokens, + }), + ).toEqual( + truncateConversationIfNeeded({ + messages, + totalTokens: belowThreshold, + contextWindow: modelInfo2.contextWindow, + maxTokens: modelInfo2.maxTokens, + }), ) // Test above threshold const aboveThreshold = 70001 - expect(truncateConversationIfNeeded(messages, aboveThreshold, modelInfo1)).toEqual( - truncateConversationIfNeeded(messages, aboveThreshold, modelInfo2), + expect( + truncateConversationIfNeeded({ + messages, + totalTokens: aboveThreshold, + contextWindow: modelInfo1.contextWindow, + maxTokens: modelInfo1.maxTokens, + }), + ).toEqual( + truncateConversationIfNeeded({ + messages, + totalTokens: aboveThreshold, + contextWindow: modelInfo2.contextWindow, + maxTokens: modelInfo2.maxTokens, + }), ) }) }) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index a0fff05ea5..8b646f933b 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,7 +1,5 @@ import { Anthropic } from "@anthropic-ai/sdk" -import { ModelInfo } from "../../shared/api" - /** * Truncates a conversation by removing a fraction of the messages. * @@ -26,28 +24,29 @@ export function truncateConversation( } /** - * Conditionally truncates the conversation messages if the total token count exceeds the model's limit. + * Conditionally truncates the conversation messages if the total token count + * exceeds the model's limit. * * @param {Anthropic.Messages.MessageParam[]} messages - The conversation messages. * @param {number} totalTokens - The total number of tokens in the conversation. - * @param {ModelInfo} modelInfo - Model metadata including context window size. + * @param {number} contextWindow - The context window size. + * @param {number} maxTokens - The maximum number of tokens allowed. * @returns {Anthropic.Messages.MessageParam[]} The original or truncated conversation messages. */ -export function truncateConversationIfNeeded( - messages: Anthropic.Messages.MessageParam[], - totalTokens: number, - modelInfo: ModelInfo, -): Anthropic.Messages.MessageParam[] { - return totalTokens < getMaxTokens(modelInfo) ? messages : truncateConversation(messages, 0.5) + +type TruncateOptions = { + messages: Anthropic.Messages.MessageParam[] + totalTokens: number + contextWindow: number + maxTokens?: number } -/** - * Calculates the maximum allowed tokens - * - * @param {ModelInfo} modelInfo - The model information containing the context window size. - * @returns {number} The maximum number of tokens allowed - */ -function getMaxTokens(modelInfo: ModelInfo): number { - // The buffer needs to be at least as large as `modelInfo.maxTokens`, or 20% of the context window if for some reason it's not set. - return modelInfo.contextWindow - (modelInfo.maxTokens || modelInfo.contextWindow * 0.2) +export function truncateConversationIfNeeded({ + messages, + totalTokens, + contextWindow, + maxTokens, +}: TruncateOptions): Anthropic.Messages.MessageParam[] { + const allowedTokens = contextWindow - (maxTokens || contextWindow * 0.2) + return totalTokens < allowedTokens ? messages : truncateConversation(messages, 0.5) } diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index bc6f457868..5e6170e2ee 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1671,6 +1671,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { requestyModelId, requestyModelInfo, modelTemperature, + modelMaxTokens, } = apiConfiguration await Promise.all([ this.updateGlobalState("apiProvider", apiProvider), @@ -1719,6 +1720,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.updateGlobalState("requestyModelId", requestyModelId), this.updateGlobalState("requestyModelInfo", requestyModelInfo), this.updateGlobalState("modelTemperature", modelTemperature), + this.updateGlobalState("modelMaxTokens", modelMaxTokens), ]) if (this.cline) { this.cline.api = buildApiHandler(apiConfiguration) @@ -2210,6 +2212,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { requestyModelId, requestyModelInfo, modelTemperature, + modelMaxTokens, maxOpenTabsContext, ] = await Promise.all([ this.getGlobalState("apiProvider") as Promise, @@ -2293,6 +2296,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.getGlobalState("requestyModelId") as Promise, this.getGlobalState("requestyModelInfo") as Promise, this.getGlobalState("modelTemperature") as Promise, + this.getGlobalState("modelMaxTokens") as Promise, this.getGlobalState("maxOpenTabsContext") as Promise, ]) @@ -2358,6 +2362,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { requestyModelId, requestyModelInfo, modelTemperature, + modelMaxTokens, }, lastShownAnnouncementId, customInstructions, diff --git a/src/shared/api.ts b/src/shared/api.ts index 5d4b8b120d..e7e4c54db6 100644 --- a/src/shared/api.ts +++ b/src/shared/api.ts @@ -68,6 +68,7 @@ export interface ApiHandlerOptions { requestyModelId?: string requestyModelInfo?: ModelInfo modelTemperature?: number + modelMaxTokens?: number } export type ApiConfiguration = ApiHandlerOptions & { @@ -92,19 +93,13 @@ export interface ModelInfo { thinking?: boolean } -export const THINKING_BUDGET = { - step: 1024, - min: 1024, - default: 8 * 1024, -} - // Anthropic // https://docs.anthropic.com/en/docs/about-claude/models export type AnthropicModelId = keyof typeof anthropicModels export const anthropicDefaultModelId: AnthropicModelId = "claude-3-7-sonnet-20250219" export const anthropicModels = { "claude-3-7-sonnet-20250219:thinking": { - maxTokens: 16384, + maxTokens: 64_000, contextWindow: 200_000, supportsImages: true, supportsComputerUse: true, @@ -116,7 +111,7 @@ export const anthropicModels = { thinking: true, }, "claude-3-7-sonnet-20250219": { - maxTokens: 16384, + maxTokens: 64_000, contextWindow: 200_000, supportsImages: true, supportsComputerUse: true, diff --git a/src/shared/globalState.ts b/src/shared/globalState.ts index 7b6b4f8274..2cc90456a7 100644 --- a/src/shared/globalState.ts +++ b/src/shared/globalState.ts @@ -81,5 +81,6 @@ export type GlobalStateKey = | "requestyModelInfo" | "unboundModelInfo" | "modelTemperature" + | "modelMaxTokens" | "mistralCodestralUrl" | "maxOpenTabsContext" diff --git a/webview-ui/src/components/settings/ThinkingBudget.tsx b/webview-ui/src/components/settings/ThinkingBudget.tsx index efaa90dc39..5b67874410 100644 --- a/webview-ui/src/components/settings/ThinkingBudget.tsx +++ b/webview-ui/src/components/settings/ThinkingBudget.tsx @@ -1,6 +1,8 @@ +import { useEffect } from "react" + import { Slider } from "@/components/ui" -import { ApiConfiguration, ModelInfo, THINKING_BUDGET } from "../../../../src/shared/api" +import { ApiConfiguration, ModelInfo } from "../../../../src/shared/api" interface ThinkingBudgetProps { apiConfiguration: ApiConfiguration @@ -9,21 +11,52 @@ interface ThinkingBudgetProps { } export const ThinkingBudget = ({ apiConfiguration, setApiConfigurationField, modelInfo }: ThinkingBudgetProps) => { - const budget = apiConfiguration?.anthropicThinking ?? THINKING_BUDGET.default + const tokens = apiConfiguration?.modelMaxTokens || modelInfo?.maxTokens || 64_000 + const tokensMin = 8192 + const tokensMax = modelInfo?.maxTokens || 64_000 - return modelInfo && modelInfo.thinking ? ( -
-
Thinking Budget
-
- setApiConfigurationField("anthropicThinking", value[0])} - /> -
{budget}
+ const thinkingTokens = apiConfiguration?.anthropicThinking || 8192 + const thinkingTokensMin = 1024 + const thinkingTokensMax = Math.floor(0.8 * tokens) + + useEffect(() => { + if (thinkingTokens > thinkingTokensMax) { + setApiConfigurationField("anthropicThinking", thinkingTokensMax) + } + }, [thinkingTokens, thinkingTokensMax, setApiConfigurationField]) + + if (!modelInfo || !modelInfo.thinking) { + return null + } + + return ( +
+
+
Max Tokens
+
+ setApiConfigurationField("modelMaxTokens", value)} + /> +
{tokens}
+
+
+
+
Max Thinking Tokens
+
+ setApiConfigurationField("anthropicThinking", value)} + /> +
{thinkingTokens}
+
- ) : null + ) }