From 764d963c104be512187ee9cb600029952c58d0a0 Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Sat, 1 Mar 2025 08:49:10 -0500 Subject: [PATCH 1/5] Add a 5k token buffer before the end of the context window --- .../__tests__/sliding-window.test.ts | 54 +++++++++++++------ src/core/sliding-window/index.ts | 6 ++- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/core/sliding-window/__tests__/sliding-window.test.ts b/src/core/sliding-window/__tests__/sliding-window.test.ts index 74b734d738..dbc0c678c2 100644 --- a/src/core/sliding-window/__tests__/sliding-window.test.ts +++ b/src/core/sliding-window/__tests__/sliding-window.test.ts @@ -3,7 +3,7 @@ import { Anthropic } from "@anthropic-ai/sdk" import { ModelInfo } from "../../../shared/api" -import { estimateTokenCount, truncateConversation, truncateConversationIfNeeded } from "../index" +import { TOKEN_BUFFER, estimateTokenCount, truncateConversation, truncateConversationIfNeeded } from "../index" /** * Tests for the truncateConversation function @@ -121,10 +121,10 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] - // Below max tokens - no truncation + // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 49999, + totalTokens: 44999, // Well below threshold + buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -133,7 +133,7 @@ describe("getMaxTokens", () => { // Above max tokens - truncate const result2 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 50001, + totalTokens: 50001, // Above threshold contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -148,10 +148,10 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] - // Below max tokens - no truncation + // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 79999, + totalTokens: 74999, // Well below threshold + buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -160,7 +160,7 @@ describe("getMaxTokens", () => { // Above max tokens - truncate const result2 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 80001, + totalTokens: 80001, // Above threshold contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -175,10 +175,10 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] - // Below max tokens - no truncation + // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 39999, + totalTokens: 34999, // Well below threshold + buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -187,7 +187,7 @@ describe("getMaxTokens", () => { // Above max tokens - truncate const result2 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 40001, + totalTokens: 40001, // Above threshold contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -202,10 +202,10 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] - // Below max tokens - no truncation + // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 169999, + totalTokens: 164999, // Well below threshold + buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -214,7 +214,7 @@ describe("getMaxTokens", () => { // Above max tokens - truncate const result2 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 170001, + totalTokens: 170001, // Above threshold contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -244,7 +244,7 @@ describe("truncateConversationIfNeeded", () => { it("should not truncate if tokens are below max tokens threshold", () => { const modelInfo = createModelInfo(100000, true, 30000) const maxTokens = 100000 - 30000 // 70000 - const totalTokens = 69999 // Below threshold + const totalTokens = 64999 // Well below threshold + buffer // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] @@ -337,8 +337,8 @@ describe("truncateConversationIfNeeded", () => { { role: messages[messages.length - 1].role, content: smallContent }, ] - // Set base tokens so total is below threshold even with small content added - const baseTokensForSmall = availableTokens - smallContentTokens - 10 + // Set base tokens so total is well below threshold + buffer even with small content added + const baseTokensForSmall = availableTokens - smallContentTokens - TOKEN_BUFFER - 10 const resultWithSmall = truncateConversationIfNeeded({ messages: messagesWithSmallContent, totalTokens: baseTokensForSmall, @@ -388,7 +388,29 @@ describe("truncateConversationIfNeeded", () => { }) expect(resultWithVeryLarge).not.toEqual(messagesWithVeryLargeContent) // Should truncate }) + + it("should truncate if tokens are within TOKEN_BUFFER of the threshold", () => { + const modelInfo = createModelInfo(100000, true, 30000) + const maxTokens = 100000 - 30000 // 70000 + const totalTokens = 66000 // Within 5000 of threshold (70000) + + // Create messages with very small content in the last one to avoid token overflow + const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] + + // When truncating, always uses 0.5 fraction + // With 4 messages after the first, 0.5 fraction means remove 2 messages + const expectedResult = [messagesWithSmallContent[0], messagesWithSmallContent[3], messagesWithSmallContent[4]] + + const result = truncateConversationIfNeeded({ + messages: messagesWithSmallContent, + totalTokens, + contextWindow: modelInfo.contextWindow, + maxTokens: modelInfo.maxTokens, + }) + expect(result).toEqual(expectedResult) + }) }) + /** * Tests for the estimateTokenCount function */ diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index 0fb26ac38f..d12e7f337e 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -3,7 +3,8 @@ import { Anthropic } from "@anthropic-ai/sdk" import { Tiktoken } from "js-tiktoken/lite" import o200kBase from "js-tiktoken/ranks/o200k_base" -const TOKEN_FUDGE_FACTOR = 1.5 +export const TOKEN_FUDGE_FACTOR = 1.5 +export const TOKEN_BUFFER = 5000 /** * Counts tokens for user content using tiktoken for text @@ -110,5 +111,6 @@ export function truncateConversationIfNeeded({ const allowedTokens = contextWindow - reservedTokens // Determine if truncation is needed and apply if necessary - return effectiveTokens < allowedTokens ? messages : truncateConversation(messages, 0.5) + // Truncate if we're within TOKEN_BUFFER of the limit + return effectiveTokens > allowedTokens - TOKEN_BUFFER ? truncateConversation(messages, 0.5) : messages } From a0684454a22e708bc3c0d79cd2293f10fb2c98b9 Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Sat, 1 Mar 2025 10:10:24 -0500 Subject: [PATCH 2/5] Add a dynamic token buffer --- .changeset/swift-lamps-decide.md | 5 ++++ .../__tests__/sliding-window.test.ts | 27 +++++++++++++------ src/core/sliding-window/index.ts | 11 +++++--- 3 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 .changeset/swift-lamps-decide.md diff --git a/.changeset/swift-lamps-decide.md b/.changeset/swift-lamps-decide.md new file mode 100644 index 0000000000..b2e7ed3e60 --- /dev/null +++ b/.changeset/swift-lamps-decide.md @@ -0,0 +1,5 @@ +--- +"roo-cline": patch +--- + +Add a dynamic token buffer diff --git a/src/core/sliding-window/__tests__/sliding-window.test.ts b/src/core/sliding-window/__tests__/sliding-window.test.ts index dbc0c678c2..df698d955a 100644 --- a/src/core/sliding-window/__tests__/sliding-window.test.ts +++ b/src/core/sliding-window/__tests__/sliding-window.test.ts @@ -3,7 +3,12 @@ import { Anthropic } from "@anthropic-ai/sdk" import { ModelInfo } from "../../../shared/api" -import { TOKEN_BUFFER, estimateTokenCount, truncateConversation, truncateConversationIfNeeded } from "../index" +import { + TOKEN_BUFFER_PERCENTAGE, + estimateTokenCount, + truncateConversation, + truncateConversationIfNeeded, +} from "../index" /** * Tests for the truncateConversation function @@ -121,10 +126,11 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] + // Account for the dynamic buffer which is 10% of context window (10,000 tokens) // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 44999, // Well below threshold + buffer + totalTokens: 39999, // Well below threshold + dynamic buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -148,10 +154,11 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] + // Account for the dynamic buffer which is 10% of context window (10,000 tokens) // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 74999, // Well below threshold + buffer + totalTokens: 69999, // Well below threshold + dynamic buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -202,10 +209,11 @@ describe("getMaxTokens", () => { // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] + // Account for the dynamic buffer which is 10% of context window (20,000 tokens for this test) // Below max tokens and buffer - no truncation const result1 = truncateConversationIfNeeded({ messages: messagesWithSmallContent, - totalTokens: 164999, // Well below threshold + buffer + totalTokens: 149999, // Well below threshold + dynamic buffer contextWindow: modelInfo.contextWindow, maxTokens: modelInfo.maxTokens, }) @@ -244,7 +252,8 @@ describe("truncateConversationIfNeeded", () => { it("should not truncate if tokens are below max tokens threshold", () => { const modelInfo = createModelInfo(100000, true, 30000) const maxTokens = 100000 - 30000 // 70000 - const totalTokens = 64999 // Well below threshold + buffer + const dynamicBuffer = modelInfo.contextWindow * TOKEN_BUFFER_PERCENTAGE // 10000 + const totalTokens = 70000 - dynamicBuffer - 1 // Just below threshold - buffer // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] @@ -338,7 +347,8 @@ describe("truncateConversationIfNeeded", () => { ] // Set base tokens so total is well below threshold + buffer even with small content added - const baseTokensForSmall = availableTokens - smallContentTokens - TOKEN_BUFFER - 10 + const dynamicBuffer = modelInfo.contextWindow * TOKEN_BUFFER_PERCENTAGE + const baseTokensForSmall = availableTokens - smallContentTokens - dynamicBuffer - 10 const resultWithSmall = truncateConversationIfNeeded({ messages: messagesWithSmallContent, totalTokens: baseTokensForSmall, @@ -389,10 +399,11 @@ describe("truncateConversationIfNeeded", () => { expect(resultWithVeryLarge).not.toEqual(messagesWithVeryLargeContent) // Should truncate }) - it("should truncate if tokens are within TOKEN_BUFFER of the threshold", () => { + it("should truncate if tokens are within TOKEN_BUFFER_PERCENTAGE of the threshold", () => { const modelInfo = createModelInfo(100000, true, 30000) const maxTokens = 100000 - 30000 // 70000 - const totalTokens = 66000 // Within 5000 of threshold (70000) + const dynamicBuffer = modelInfo.contextWindow * TOKEN_BUFFER_PERCENTAGE // 10% of 100000 = 10000 + const totalTokens = 70000 - dynamicBuffer + 1 // Just within the dynamic buffer of threshold (70000) // Create messages with very small content in the last one to avoid token overflow const messagesWithSmallContent = [...messages.slice(0, -1), { ...messages[messages.length - 1], content: "" }] diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts index d12e7f337e..48548ecfc7 100644 --- a/src/core/sliding-window/index.ts +++ b/src/core/sliding-window/index.ts @@ -4,7 +4,10 @@ import { Tiktoken } from "js-tiktoken/lite" import o200kBase from "js-tiktoken/ranks/o200k_base" export const TOKEN_FUDGE_FACTOR = 1.5 -export const TOKEN_BUFFER = 5000 +/** + * Default percentage of the context window to use as a buffer when deciding when to truncate + */ +export const TOKEN_BUFFER_PERCENTAGE = 0.1 /** * Counts tokens for user content using tiktoken for text @@ -108,9 +111,9 @@ export function truncateConversationIfNeeded({ const effectiveTokens = totalTokens + lastMessageTokens // Calculate available tokens for conversation history - const allowedTokens = contextWindow - reservedTokens + // Truncate if we're within TOKEN_BUFFER_PERCENTAGE of the context window + const allowedTokens = contextWindow * (1 - TOKEN_BUFFER_PERCENTAGE) - reservedTokens // Determine if truncation is needed and apply if necessary - // Truncate if we're within TOKEN_BUFFER of the limit - return effectiveTokens > allowedTokens - TOKEN_BUFFER ? truncateConversation(messages, 0.5) : messages + return effectiveTokens > allowedTokens ? truncateConversation(messages, 0.5) : messages } From 163528eee99f96b4f103ecbd2d7d630ab65a7e43 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 1 Mar 2025 15:15:46 +0000 Subject: [PATCH 3/5] changeset version bump --- .changeset/chilly-bugs-pay.md | 5 ----- .changeset/swift-lamps-decide.md | 5 ----- .changeset/tasty-grapes-suffer.md | 5 ----- .changeset/young-hornets-taste.md | 5 ----- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 7 files changed, 12 insertions(+), 23 deletions(-) delete mode 100644 .changeset/chilly-bugs-pay.md delete mode 100644 .changeset/swift-lamps-decide.md delete mode 100644 .changeset/tasty-grapes-suffer.md delete mode 100644 .changeset/young-hornets-taste.md diff --git a/.changeset/chilly-bugs-pay.md b/.changeset/chilly-bugs-pay.md deleted file mode 100644 index b30f8241ef..0000000000 --- a/.changeset/chilly-bugs-pay.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"roo-cline": patch ---- - -Delete task confirmation enhancements diff --git a/.changeset/swift-lamps-decide.md b/.changeset/swift-lamps-decide.md deleted file mode 100644 index b2e7ed3e60..0000000000 --- a/.changeset/swift-lamps-decide.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"roo-cline": patch ---- - -Add a dynamic token buffer diff --git a/.changeset/tasty-grapes-suffer.md b/.changeset/tasty-grapes-suffer.md deleted file mode 100644 index 7382b38c77..0000000000 --- a/.changeset/tasty-grapes-suffer.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"roo-cline": patch ---- - -Fix maxTokens defaults for Claude 3.7 Sonnet models diff --git a/.changeset/young-hornets-taste.md b/.changeset/young-hornets-taste.md deleted file mode 100644 index 1b9c3d94e8..0000000000 --- a/.changeset/young-hornets-taste.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"roo-cline": patch ---- - -Prettier thinking blocks diff --git a/CHANGELOG.md b/CHANGELOG.md index 9622ce0c99..944fd19e54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Roo Code Changelog +## 3.7.9 + +### Patch Changes + +- Delete task confirmation enhancements +- Add a dynamic token buffer +- Fix maxTokens defaults for Claude 3.7 Sonnet models +- Prettier thinking blocks + ## [3.7.8] - Add Vertex AI prompt caching support for Claude models (thanks @aitoroses and @lupuletic!) diff --git a/package-lock.json b/package-lock.json index 0b24ce6664..72449b850a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "roo-cline", - "version": "3.7.8", + "version": "3.7.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "roo-cline", - "version": "3.7.8", + "version": "3.7.9", "dependencies": { "@anthropic-ai/bedrock-sdk": "^0.10.2", "@anthropic-ai/sdk": "^0.37.0", diff --git a/package.json b/package.json index 6f5699a7c7..b8427fd303 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "Roo Code (prev. Roo Cline)", "description": "A whole dev team of AI agents in your editor.", "publisher": "RooVeterinaryInc", - "version": "3.7.8", + "version": "3.7.9", "icon": "assets/icons/rocket.png", "galleryBanner": { "color": "#617A91", From 574348d9639202e7eebb4f5ecb6227ebbd3741ba Mon Sep 17 00:00:00 2001 From: Matt Rubens Date: Sat, 1 Mar 2025 10:22:13 -0500 Subject: [PATCH 4/5] Update CHANGELOG.md --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 944fd19e54..691dbbe079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ # Roo Code Changelog -## 3.7.9 - -### Patch Changes +## [3.7.9] - Delete task confirmation enhancements -- Add a dynamic token buffer -- Fix maxTokens defaults for Claude 3.7 Sonnet models +- Smarter context window management - Prettier thinking blocks +- Fix maxTokens defaults for Claude 3.7 Sonnet models +- Terminal output parsing improvements (thanks @KJ7LNW!) +- UI fix to dropdown hover colors (thanks @SamirSaji!) +- Add support for Claude Sonnet 3.7 thinking via Vertex AI (thanks @lupuletic!) ## [3.7.8] From f4a6f06a28d40c5fcc39346f7657921588dbb619 Mon Sep 17 00:00:00 2001 From: Chris Estreich Date: Sat, 1 Mar 2025 08:18:43 -0800 Subject: [PATCH 5/5] Fix "install dependencies" section of marketplace-publish --- .github/workflows/marketplace-publish.yml | 5 +---- package.json | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/marketplace-publish.yml b/.github/workflows/marketplace-publish.yml index fcc089c1db..794e598b80 100644 --- a/.github/workflows/marketplace-publish.yml +++ b/.github/workflows/marketplace-publish.yml @@ -29,10 +29,7 @@ jobs: - name: Install Dependencies run: | npm install -g vsce ovsx - npm install - cd webview-ui - npm install - cd .. + npm run install:ci - name: Package and Publish Extension env: VSCE_PAT: ${{ secrets.VSCE_PAT }} diff --git a/package.json b/package.json index b8427fd303..16b1e1e5d4 100644 --- a/package.json +++ b/package.json @@ -334,6 +334,7 @@ "get-folder-size": "^5.0.0", "globby": "^14.0.2", "isbinaryfile": "^5.0.2", + "js-tiktoken": "^1.0.19", "mammoth": "^1.8.0", "monaco-vscode-textmate-theme-converter": "^0.1.7", "openai": "^4.78.1", @@ -348,7 +349,6 @@ "sound-play": "^1.1.0", "string-similarity": "^4.0.4", "strip-ansi": "^7.1.0", - "js-tiktoken": "^1.0.19", "tmp": "^0.2.3", "tree-sitter-wasms": "^0.1.11", "turndown": "^7.2.0",