From 7386e3db2b774caddf032075b18a6d23129e9a6f Mon Sep 17 00:00:00 2001
From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com>
Date: Thu, 26 Dec 2024 13:22:13 -0800
Subject: [PATCH] Add OpenRouter middle out transform to avoid context limit
 errors

---
 src/api/providers/openrouter.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts
index 93bbaba87b..8d37cdf0a3 100644
--- a/src/api/providers/openrouter.ts
+++ b/src/api/providers/openrouter.ts
@@ -95,12 +95,18 @@ export class OpenRouterHandler implements ApiHandler {
 				maxTokens = 8_192
 				break
 		}
+
+		// Removes messages in the middle when close to context window limit. Should not be applied to models that support prompt caching since it would continuously break the cache.
+		const shouldApplyMiddleOutTransform = !this.getModel().info.supportsPromptCache
+
+		// @ts-ignore-next-line
 		const stream = await this.client.chat.completions.create({
 			model: this.getModel().id,
 			max_tokens: maxTokens,
 			temperature: 0,
 			messages: openAiMessages,
 			stream: true,
+			transforms: shouldApplyMiddleOutTransform ? ["middle-out"] : undefined,
 		})
 
 		let genId: string | undefined