From d154d059a607e61badf6528e29d9e033a2427fe7 Mon Sep 17 00:00:00 2001 From: Nissa Seru <119150866+nissa-seru@users.noreply.github.com> Date: Fri, 31 Jan 2025 00:40:50 -0500 Subject: [PATCH 1/5] Enable separate config for truncation for models without context caching --- src/core/Cline.ts | 33 ++++++++++++-------- src/core/sliding-window/index.ts | 52 +++++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 0c0bd37050..3299b0cab0 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -53,7 +53,7 @@ import { AssistantMessageContent, parseAssistantMessage, ToolParamName, ToolUseN import { formatResponse } from "./prompts/responses" import { SYSTEM_PROMPT } from "./prompts/system" import { modes, defaultModeSlug, getModeBySlug } from "../shared/modes" -import { truncateHalfConversation } from "./sliding-window" +import { truncateConversationIfNeeded } from "./sliding-window" import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider" import { detectCodeOmission } from "../integrations/editor/detect-omission" import { BrowserSession } from "../services/browser/BrowserSession" @@ -837,18 +837,25 @@ export class Cline { // If the previous API request's total token usage is close to the context window, truncate the conversation history to free up space for the new request if (previousApiReqIndex >= 0) { - const previousRequest = this.clineMessages[previousApiReqIndex] - if (previousRequest && previousRequest.text) { - const { tokensIn, tokensOut, cacheWrites, cacheReads }: ClineApiReqInfo = JSON.parse( - previousRequest.text, - ) - const totalTokens = (tokensIn || 0) + (tokensOut || 0) + (cacheWrites || 0) + (cacheReads || 0) - const contextWindow = this.api.getModel().info.contextWindow || 128_000 - const maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8) - if (totalTokens >= maxAllowedSize) { - const truncatedMessages = truncateHalfConversation(this.apiConversationHistory) - await this.overwriteApiConversationHistory(truncatedMessages) - } + const previousRequest = this.clineMessages[previousApiReqIndex]?.text + if (!previousRequest) return + + const { + tokensIn = 0, + tokensOut = 0, + cacheWrites = 0, + cacheReads = 0, + }: ClineApiReqInfo = JSON.parse(previousRequest) + const totalTokens = tokensIn + tokensOut + cacheWrites + cacheReads + + const trimmedMessages = truncateConversationIfNeeded( + this.apiConversationHistory, + totalTokens, + this.api.getModel().info, + ) + + if (trimmedMessages !== this.apiConversationHistory) { + await this.overwriteApiConversationHistory(trimmedMessages) } } diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index caa604bc57..981d02fbae 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,4 +1,5 @@ import { Anthropic } from "@anthropic-ai/sdk" +<<<<<<< HEAD /* We can't implement a dynamically updating sliding window as it would break prompt cache @@ -9,18 +10,53 @@ Therefore, this function should only be called when absolutely necessary to fit context limits, not as a continuous process. */ export function truncateHalfConversation( +======= +import { ModelInfo } from "../../shared/api" +import { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs" + +export function truncateConversation( +>>>>>>> 455d850c (Enable separate config for truncation for models without context caching) messages: Anthropic.Messages.MessageParam[], + fracToRemove: number, ): Anthropic.Messages.MessageParam[] { - // API expects messages to be in user-assistant order, and tool use messages must be followed by tool results. We need to maintain this structure while truncating. - - // Always keep the first Task message (this includes the project's file structure in environment_details) const truncatedMessages = [messages[0]] - - // Remove half of user-assistant pairs - const messagesToRemove = Math.floor(messages.length / 4) * 2 // has to be even number - - const remainingMessages = messages.slice(messagesToRemove + 1) // has to start with assistant message since tool result cannot follow assistant message with no tool use + const rawMessagesToRemove = Math.floor((messages.length - 1) * fracToRemove) + const messagesToRemove = rawMessagesToRemove - (rawMessagesToRemove % 2) + const remainingMessages = messages.slice(messagesToRemove + 1) truncatedMessages.push(...remainingMessages) return truncatedMessages } + +export function truncateConversationIfNeeded( + messages: MessageParam[], + totalTokens: number, + modelInfo: ModelInfo, +): MessageParam[] { + if (modelInfo.supportsPromptCache) { + return totalTokens < getMaxTokensForPromptCachingModels(modelInfo) + ? messages + : truncateConversation(messages, getTruncFractionForPromptCachingModels(modelInfo)) + } else { + const thresh = getMaxTokensForNonPromptCachingModels(modelInfo) + return totalTokens < thresh + ? messages + : truncateConversation(messages, getTruncFractionForNonPromptCachingModels(modelInfo)) + } +} + +function getMaxTokensForPromptCachingModels(modelInfo: ModelInfo): number { + return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) +} + +function getTruncFractionForPromptCachingModels(modelInfo: ModelInfo): number { + return Math.min(80_000, modelInfo.contextWindow * 0.4) +} + +function getMaxTokensForNonPromptCachingModels(modelInfo: ModelInfo): number { + return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) +} + +function getTruncFractionForNonPromptCachingModels(modelInfo: ModelInfo): number { + return Math.min(80_000, modelInfo.contextWindow * 0.4) +} From 9da39e42e1b72f0421480e10e867e0f3b71e8da9 Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Sun, 2 Feb 2025 01:23:45 -0500 Subject: [PATCH 2/5] Fix conflict --- src/core/sliding-window/index.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 981d02fbae..6fbd5b8a9c 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,21 +1,8 @@ import { Anthropic } from "@anthropic-ai/sdk" -<<<<<<< HEAD - -/* -We can't implement a dynamically updating sliding window as it would break prompt cache -every time. To maintain the benefits of caching, we need to keep conversation history -static. This operation should be performed as infrequently as possible. If a user reaches -a 200k context, we can assume that the first half is likely irrelevant to their current task. -Therefore, this function should only be called when absolutely necessary to fit within -context limits, not as a continuous process. -*/ -export function truncateHalfConversation( -======= import { ModelInfo } from "../../shared/api" import { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs" export function truncateConversation( ->>>>>>> 455d850c (Enable separate config for truncation for models without context caching) messages: Anthropic.Messages.MessageParam[], fracToRemove: number, ): Anthropic.Messages.MessageParam[] { From fef95e10b83a9bc94d8956e68d4ebfa208188091 Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Sun, 2 Feb 2025 01:34:45 -0500 Subject: [PATCH 3/5] Fix type --- src/core/sliding-window/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 6fbd5b8a9c..3ecb785f5e 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,6 +1,5 @@ import { Anthropic } from "@anthropic-ai/sdk" import { ModelInfo } from "../../shared/api" -import { MessageParam } from "@anthropic-ai/sdk/resources/messages.mjs" export function truncateConversation( messages: Anthropic.Messages.MessageParam[], @@ -16,10 +15,10 @@ export function truncateConversation( } export function truncateConversationIfNeeded( - messages: MessageParam[], + messages: Anthropic.Messages.MessageParam[], totalTokens: number, modelInfo: ModelInfo, -): MessageParam[] { +): Anthropic.Messages.MessageParam[] { if (modelInfo.supportsPromptCache) { return totalTokens < getMaxTokensForPromptCachingModels(modelInfo) ? messages From 3492f79f37b1ba81d19af080ea10172a0364aac7 Mon Sep 17 00:00:00 2001 From: Nissa Seru <119150866+nissa-seru@users.noreply.github.com> Date: Sun, 2 Feb 2025 02:56:30 -0500 Subject: [PATCH 4/5] Remove unnecessary variable, add JSDocs --- src/core/sliding-window/index.ts | 57 +++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 3ecb785f5e..9e98ba17d7 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -1,6 +1,16 @@ import { Anthropic } from "@anthropic-ai/sdk" import { ModelInfo } from "../../shared/api" +/** + * Truncates a conversation by removing a fraction of the messages. + * + * The first message is always retained, and a specified fraction (rounded to an even number) + * of messages from the beginning (excluding the first) is removed. + * + * @param {Anthropic.Messages.MessageParam[]} messages - The conversation messages. + * @param {number} fracToRemove - The fraction (between 0 and 1) of messages (excluding the first) to remove. + * @returns {Anthropic.Messages.MessageParam[]} The truncated conversation messages. + */ export function truncateConversation( messages: Anthropic.Messages.MessageParam[], fracToRemove: number, @@ -14,6 +24,18 @@ export function truncateConversation( return truncatedMessages } +/** + * Conditionally truncates the conversation messages if the total token count exceeds the model's limit. + * + * Depending on whether the model supports prompt caching, different maximum token thresholds + * and truncation fractions are used. If the current total tokens exceed the threshold, + * the conversation is truncated using the appropriate fraction. + * + * @param {Anthropic.Messages.MessageParam[]} messages - The conversation messages. + * @param {number} totalTokens - The total number of tokens in the conversation. + * @param {ModelInfo} modelInfo - Model metadata including context window size and prompt cache support. + * @returns {Anthropic.Messages.MessageParam[]} The original or truncated conversation messages. + */ export function truncateConversationIfNeeded( messages: Anthropic.Messages.MessageParam[], totalTokens: number, @@ -24,25 +46,52 @@ export function truncateConversationIfNeeded( ? messages : truncateConversation(messages, getTruncFractionForPromptCachingModels(modelInfo)) } else { - const thresh = getMaxTokensForNonPromptCachingModels(modelInfo) - return totalTokens < thresh + return totalTokens < getMaxTokensForNonPromptCachingModels(modelInfo) ? messages : truncateConversation(messages, getTruncFractionForNonPromptCachingModels(modelInfo)) } } +/** + * Calculates the maximum allowed tokens for models that support prompt caching. + * + * The maximum is computed as the greater of (contextWindow - 40000) and 80% of the contextWindow. + * + * @param {ModelInfo} modelInfo - The model information containing the context window size. + * @returns {number} The maximum number of tokens allowed for prompt caching models. + */ function getMaxTokensForPromptCachingModels(modelInfo: ModelInfo): number { return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) } +/** + * Provides the fraction of messages to remove for models that support prompt caching. + * + * @param {ModelInfo} modelInfo - The model information (unused in current implementation). + * @returns {number} The truncation fraction for prompt caching models (fixed at 0.5). + */ function getTruncFractionForPromptCachingModels(modelInfo: ModelInfo): number { - return Math.min(80_000, modelInfo.contextWindow * 0.4) + return 0.5 } +/** + * Calculates the maximum allowed tokens for models that do not support prompt caching. + * + * The maximum is computed as the greater of (contextWindow - 40000) and 80% of the contextWindow. + * + * @param {ModelInfo} modelInfo - The model information containing the context window size. + * @returns {number} The maximum number of tokens allowed for non-prompt caching models. + */ function getMaxTokensForNonPromptCachingModels(modelInfo: ModelInfo): number { return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) } +/** + * Provides the fraction of messages to remove for models that do not support prompt caching. + * + * @param {ModelInfo} modelInfo - The model information (unused in current implementation). + * @returns {number} The truncation fraction for non-prompt caching models (fixed at 0.1). + */ function getTruncFractionForNonPromptCachingModels(modelInfo: ModelInfo): number { - return Math.min(80_000, modelInfo.contextWindow * 0.4) + return 0.1 } From 98c8cc9624c9a796e1a22e428ee40101599259cc Mon Sep 17 00:00:00 2001 From: Nissa Seru <119150866+nissa-seru@users.noreply.github.com> Date: Wed, 5 Feb 2025 04:53:21 -0500 Subject: [PATCH 5/5] Updated sliding window logic to be inverse of trigger criteria --- src/core/sliding-window/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 9e98ba17d7..ee4a1543e7 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -89,9 +89,9 @@ function getMaxTokensForNonPromptCachingModels(modelInfo: ModelInfo): number { /** * Provides the fraction of messages to remove for models that do not support prompt caching. * - * @param {ModelInfo} modelInfo - The model information (unused in current implementation). + * @param {ModelInfo} modelInfo - The model information. * @returns {number} The truncation fraction for non-prompt caching models (fixed at 0.1). */ function getTruncFractionForNonPromptCachingModels(modelInfo: ModelInfo): number { - return 0.1 + return Math.min(40_000 / modelInfo.contextWindow, 0.2) }