From 1d67f88aba2ccd1ab77343202c57799353a6f082 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 24 Feb 2025 15:09:46 -0800 Subject: [PATCH] Sliding window fix --- src/core/sliding-window/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index ee4a1543e7..c84f308387 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -55,13 +55,15 @@ export function truncateConversationIfNeeded( /** * Calculates the maximum allowed tokens for models that support prompt caching. * - * The maximum is computed as the greater of (contextWindow - 40000) and 80% of the contextWindow. + * The maximum is computed as the greater of (contextWindow - buffer) and 80% of the contextWindow. * * @param {ModelInfo} modelInfo - The model information containing the context window size. * @returns {number} The maximum number of tokens allowed for prompt caching models. */ function getMaxTokensForPromptCachingModels(modelInfo: ModelInfo): number { - return Math.max(modelInfo.contextWindow - 40_000, modelInfo.contextWindow * 0.8) + // The buffer needs to be at least as large as `modelInfo.maxTokens`. + const buffer = modelInfo.maxTokens ? Math.max(40_000, modelInfo.maxTokens) : 40_000 + return Math.max(modelInfo.contextWindow - buffer, modelInfo.contextWindow * 0.8) } /**