From 7e62d34570583c3f2c7264da9e1b6ba502eb0b27 Mon Sep 17 00:00:00 2001
From: Matt Rubens <mrubens@users.noreply.github.com>
Date: Sun, 2 Mar 2025 02:09:36 -0500
Subject: [PATCH] PR cleanup

---
 src/api/providers/base-provider.ts   | 16 +++++++++++-----
 src/core/sliding-window/constants.ts |  9 ---------
 src/core/sliding-window/index.ts     |  7 ++++---
 3 files changed, 15 insertions(+), 17 deletions(-)
 delete mode 100644 src/core/sliding-window/constants.ts

diff --git a/src/api/providers/base-provider.ts b/src/api/providers/base-provider.ts
index e44d98e7d1..34156e4adf 100644
--- a/src/api/providers/base-provider.ts
+++ b/src/api/providers/base-provider.ts
@@ -12,6 +12,8 @@ const TOKEN_FUDGE_FACTOR = 1.5
  * Base class for API providers that implements common functionality
  */
 export abstract class BaseProvider implements ApiHandler {
+	// Cache the Tiktoken encoder instance since it's stateless
+	private encoder: Tiktoken | null = null
 	abstract createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
 	abstract getModel(): { id: string; info: ModelInfo }
 
@@ -19,6 +21,9 @@ export abstract class BaseProvider implements ApiHandler {
 	 * Default token counting implementation using tiktoken
 	 * Providers can override this to use their native token counting endpoints
 	 *
+	 * Uses a cached Tiktoken encoder instance for performance since it's stateless.
+	 * The encoder is created lazily on first use and reused for subsequent calls.
+	 *
 	 * @param content The content to count tokens for
 	 * @returns A promise resolving to the token count
 	 */
@@ -27,17 +32,18 @@ export abstract class BaseProvider implements ApiHandler {
 
 		let totalTokens = 0
 
-		// Create encoder - currently we only use o200kBase
-		// In the future, providers could override this method to use more specific tokenizers
-		const encoder = new Tiktoken(o200kBase)
+		// Lazily create and cache the encoder if it doesn't exist
+		if (!this.encoder) {
+			this.encoder = new Tiktoken(o200kBase)
+		}
 
-		// Process each content block
+		// Process each content block using the cached encoder
 		for (const block of content) {
 			if (block.type === "text") {
 				// Use tiktoken for text token counting
 				const text = block.text || ""
 				if (text.length > 0) {
-					const tokens = encoder.encode(text)
+					const tokens = this.encoder.encode(text)
 					totalTokens += tokens.length
 				}
 			} else if (block.type === "image") {
diff --git a/src/core/sliding-window/constants.ts b/src/core/sliding-window/constants.ts
deleted file mode 100644
index 608483a46f..0000000000
--- a/src/core/sliding-window/constants.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-/**
- * Factor to apply to token counts to account for estimation inaccuracies
- */
-export const TOKEN_FUDGE_FACTOR = 1.5
-
-/**
- * Default percentage of the context window to use as a buffer when deciding when to truncate
- */
-export const TOKEN_BUFFER_PERCENTAGE = 0.1
diff --git a/src/core/sliding-window/index.ts b/src/core/sliding-window/index.ts
index ffb347931c..67c0028fab 100644
--- a/src/core/sliding-window/index.ts
+++ b/src/core/sliding-window/index.ts
@@ -1,9 +1,10 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import { ApiHandler } from "../../api"
-import { TOKEN_BUFFER_PERCENTAGE } from "./constants"
 
-// Re-export constants for external use
-export { TOKEN_BUFFER_PERCENTAGE } from "./constants"
+/**
+ * Default percentage of the context window to use as a buffer when deciding when to truncate
+ */
+export const TOKEN_BUFFER_PERCENTAGE = 0.1
 
 /**
  * Counts tokens for user content using the provider's token counting implementation.