From 3ae121687c60fc4e2463644be0367c71984fc724 Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Fri, 7 Feb 2025 01:07:06 -0500 Subject: [PATCH] Revert "Enable separate config for truncation for models without context caching" --- src/core/Cline.ts | 33 ++++------ src/core/sliding-window/index.ts | 105 +++++-------------------------- webview-ui/tsconfig.json | 15 ++++- 3 files changed, 42 insertions(+), 111 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 2c36dd5495..39013f0cb4 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -53,7 +53,7 @@ import { AssistantMessageContent, parseAssistantMessage, ToolParamName, ToolUseN import { formatResponse } from "./prompts/responses" import { SYSTEM_PROMPT } from "./prompts/system" import { modes, defaultModeSlug, getModeBySlug } from "../shared/modes" -import { truncateConversationIfNeeded } from "./sliding-window" +import { truncateHalfConversation } from "./sliding-window" import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider" import { detectCodeOmission } from "../integrations/editor/detect-omission" import { BrowserSession } from "../services/browser/BrowserSession" @@ -876,25 +876,18 @@ export class Cline { // If the previous API request's total token usage is close to the context window, truncate the conversation history to free up space for the new request if (previousApiReqIndex >= 0) { - const previousRequest = this.clineMessages[previousApiReqIndex]?.text - if (!previousRequest) return - - const { - tokensIn = 0, - tokensOut = 0, - cacheWrites = 0, - cacheReads = 0, - }: ClineApiReqInfo = JSON.parse(previousRequest) - const totalTokens = tokensIn + tokensOut + cacheWrites + cacheReads - - const trimmedMessages = truncateConversationIfNeeded( - this.apiConversationHistory, - totalTokens, - this.api.getModel().info, - ) - - if (trimmedMessages !== this.apiConversationHistory) { - await this.overwriteApiConversationHistory(trimmedMessages) + const previousRequest = this.clineMessages[previousApiReqIndex] + if (previousRequest && previousRequest.text) { + const { tokensIn, tokensOut, cacheWrites, cacheReads }: ClineApiReqInfo = JSON.parse( + previousRequest.text, + ) + const totalTokens = (tokensIn || 0) + (tokensOut || 0) + (cacheWrites || 0) + (cacheReads || 0) + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8) + if (totalTokens >= maxAllowedSize) { + const truncatedMessages = truncateHalfConversation(this.apiConversationHistory) + await this.overwriteApiConversationHistory(truncatedMessages) + } } } diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index ee4a1543e7..caa604bc57 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,97 +1,26 @@ import { Anthropic } from "@anthropic-ai/sdk" -import { ModelInfo } from "../../shared/api" -/** - * Truncates a conversation by removing a fraction of the messages. - * - * The first message is always retained, and a specified fraction (rounded to an even number) - * of messages from the beginning (excluding the first) is removed. - * - * @param {Anthropic.Messages.MessageParam[]} messages - The conversation messages. - * @param {number} fracToRemove - The fraction (between 0 and 1) of messages (excluding the first) to remove. - * @returns {Anthropic.Messages.MessageParam[]} The truncated conversation messages. - */ -export function truncateConversation( +/* +We can't implement a dynamically updating sliding window as it would break prompt cache +every time. To maintain the benefits of caching, we need to keep conversation history +static. This operation should be performed as infrequently as possible. If a user reaches +a 200k context, we can assume that the first half is likely irrelevant to their current task. +Therefore, this function should only be called when absolutely necessary to fit within +context limits, not as a continuous process. +*/ +export function truncateHalfConversation( messages: Anthropic.Messages.MessageParam[], - fracToRemove: number, ): Anthropic.Messages.MessageParam[] { + // API expects messages to be in user-assistant order, and tool use messages must be followed by tool results. We need to maintain this structure while truncating. + + // Always keep the first Task message (this includes the project's file structure in environment_details) const truncatedMessages = [messages[0]] - const rawMessagesToRemove = Math.floor((messages.length - 1) * fracToRemove) - const messagesToRemove = rawMessagesToRemove - (rawMessagesToRemove % 2) - const remainingMessages = messages.slice(messagesToRemove + 1) + + // Remove half of user-assistant pairs + const messagesToRemove = Math.floor(messages.length / 4) * 2 // has to be even number + + const remainingMessages = messages.slice(messagesToRemove + 1) // has to start with assistant message since tool result cannot follow assistant message with no tool use truncatedMessages.push(...remainingMessages) return truncatedMessages } - -/** - * Conditionally truncates the conversation messages if the total token count exceeds the model's limit. - * - * Depending on whether the model supports prompt caching, different maximum token thresholds - * and truncation fractions are used. If the current total tokens exceed the threshold, - * the conversation is truncated using the appropriate fraction. - * - * @param {Anthropic.Messages.MessageParam[]} messages - The conversation messages. - * @param {number} totalTokens - The total number of tokens in the conversation. - * @param {ModelInfo} modelInfo - Model metadata including context window size and prompt cache support. - * @returns {Anthropic.Messages.MessageParam[]} The original or truncated conversation messages. - */ -export function truncateConversationIfNeeded( - messages: Anthropic.Messages.MessageParam[], - totalTokens: number, - modelInfo: ModelInfo, -): Anthropic.Messages.MessageParam[] { - if (modelInfo.supportsPromptCache) { - return totalTokens < getMaxTokensForPromptCachingModels(modelInfo) - ? messages - : truncateConversation(messages, getTruncFractionForPromptCachingModels(modelInfo)) - } else { - return totalTokens < getMaxTokensForNonPromptCachingModels(modelInfo) - ? messages - : truncateConversation(messages, getTruncFractionForNonPromptCachingModels(modelInfo)) - } -} - -/** - * Calculates the maximum allowed tokens for models that support prompt caching. - * - * The maximum is computed as the greater of (contextWindow - 40000) and 80% of the contextWindow. - * - * @param {ModelInfo} modelInfo - The model information containing the context window size. - * @returns {number} The maximum number of tokens allowed for prompt caching models. - */ -function getMaxTokensForPromptCachingModels(modelInfo: ModelInfo): number { - return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) -} - -/** - * Provides the fraction of messages to remove for models that support prompt caching. - * - * @param {ModelInfo} modelInfo - The model information (unused in current implementation). - * @returns {number} The truncation fraction for prompt caching models (fixed at 0.5). - */ -function getTruncFractionForPromptCachingModels(modelInfo: ModelInfo): number { - return 0.5 -} - -/** - * Calculates the maximum allowed tokens for models that do not support prompt caching. - * - * The maximum is computed as the greater of (contextWindow - 40000) and 80% of the contextWindow. - * - * @param {ModelInfo} modelInfo - The model information containing the context window size. - * @returns {number} The maximum number of tokens allowed for non-prompt caching models. - */ -function getMaxTokensForNonPromptCachingModels(modelInfo: ModelInfo): number { - return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) -} - -/** - * Provides the fraction of messages to remove for models that do not support prompt caching. - * - * @param {ModelInfo} modelInfo - The model information. - * @returns {number} The truncation fraction for non-prompt caching models (fixed at 0.1). - */ -function getTruncFractionForNonPromptCachingModels(modelInfo: ModelInfo): number { - return Math.min(40_000 / modelInfo.contextWindow, 0.2) -} diff --git a/webview-ui/tsconfig.json b/webview-ui/tsconfig.json index c725fcff3e..2243662462 100644 --- a/webview-ui/tsconfig.json +++ b/webview-ui/tsconfig.json @@ -1,7 +1,11 @@ { "compilerOptions": { "target": "es5", - "lib": ["dom", "dom.iterable", "esnext"], + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], "allowJs": true, "skipLibCheck": true, "esModuleInterop": true, @@ -17,8 +21,13 @@ "jsx": "react-jsx", "baseUrl": ".", "paths": { - "@/*": ["./src/*"] + "@/*": [ + "./src/*" + ] } }, - "include": ["src", "../src/shared"] + "include": [ + "src", + "../src/shared", + ] }