From b6fd74b3ec72bdbf56bae86abb1029ec7f97c553 Mon Sep 17 00:00:00 2001 From: Hannes Rudolph Date: Fri, 13 Feb 2026 09:25:12 -0700 Subject: [PATCH] refactor: move applyCacheBreakpoints into providers, reduce maxBreakpoints to 1 - Move applyCacheBreakpoints() from Task.ts into each provider's createMessage() (anthropic, bedrock, openrouter, requesty, anthropic-vertex, roo) - Aligns with AI SDK best practice: cache control is a call-site concern - Remove stripCacheProviderOptions() from saveRooMessages() save path (no longer needed since cache markers don't contaminate persistent history) - Change maxBreakpoints default from 2 to 1 (3-point strategy: 1 system, 1 tool, 1 message) --- src/api/providers/anthropic-vertex.ts | 4 +- src/api/providers/anthropic.ts | 4 +- src/api/providers/bedrock.ts | 4 +- src/api/providers/openrouter.ts | 4 +- src/api/providers/requesty.ts | 4 +- src/api/providers/roo.ts | 4 +- .../__tests__/cache-breakpoints.spec.ts | 39 +++++++++++-------- src/api/transform/cache-breakpoints.ts | 10 ++--- src/core/task-persistence/apiMessages.ts | 3 +- src/core/task/Task.ts | 5 +-- 10 files changed, 48 insertions(+), 33 deletions(-) diff --git a/src/api/providers/anthropic-vertex.ts b/src/api/providers/anthropic-vertex.ts index ee70152fa8..33e4fd3ab4 100644 --- a/src/api/providers/anthropic-vertex.ts +++ b/src/api/providers/anthropic-vertex.ts @@ -25,7 +25,7 @@ import { handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" -import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" import { calculateApiCostAnthropic } from "../../shared/cost" import { DEFAULT_HEADERS } from "./constants" @@ -127,6 +127,8 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple metadata?.systemProviderOptions, ) + applyCacheBreakpoints(aiSdkMessages) + // Build streamText request // Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values const requestOptions: Parameters[0] = { diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index aa080c87a8..8827eae496 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -23,7 +23,7 @@ import { handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" -import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" import { sanitizeMessagesForProvider } from "../transform/sanitize-messages" import { calculateApiCostAnthropic } from "../../shared/cost" @@ -116,6 +116,8 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa metadata?.systemProviderOptions, ) + applyCacheBreakpoints(aiSdkMessages) + // Build streamText request // Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values const requestOptions: Parameters[0] = { diff --git a/src/api/providers/bedrock.ts b/src/api/providers/bedrock.ts index c337abf03b..4d9ac15003 100644 --- a/src/api/providers/bedrock.ts +++ b/src/api/providers/bedrock.ts @@ -31,7 +31,7 @@ import { handleAiSdkError, yieldResponseMessage, } from "../transform/ai-sdk" -import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" import { getModelParams } from "../transform/model-params" import { sanitizeMessagesForProvider } from "../transform/sanitize-messages" import { shouldUseReasoningBudget } from "../../shared/api" @@ -252,6 +252,8 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH ? applySystemPromptCaching(systemPrompt, aiSdkMessages, metadata?.systemProviderOptions) : systemPrompt || undefined + applyCacheBreakpoints(aiSdkMessages) + // Strip non-Bedrock cache annotations from messages when caching is disabled, // and strip Bedrock-specific annotations when caching is disabled. if (!usePromptCache) { diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 578d36fa11..1495a8246b 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -18,7 +18,7 @@ import { calculateApiCostOpenAI } from "../../shared/cost" import { getModelParams } from "../transform/model-params" import { convertToolsForAiSdk, processAiSdkStreamPart, yieldResponseMessage } from "../transform/ai-sdk" -import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" import { BaseProvider } from "./base-provider" import { getModels, getModelsFromCache } from "./fetchers/modelCache" @@ -182,6 +182,8 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH metadata?.systemProviderOptions, ) + applyCacheBreakpoints(aiSdkMessages) + try { const result = streamText({ model: openrouter.chat(modelId), diff --git a/src/api/providers/requesty.ts b/src/api/providers/requesty.ts index b49428e4c9..85895c9422 100644 --- a/src/api/providers/requesty.ts +++ b/src/api/providers/requesty.ts @@ -8,7 +8,7 @@ import type { ApiHandlerOptions } from "../../shared/api" import { calculateApiCostOpenAI } from "../../shared/cost" import { convertToolsForAiSdk, consumeAiSdkStream, mapToolChoice, handleAiSdkError } from "../transform/ai-sdk" -import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints" import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { getModelParams } from "../transform/model-params" @@ -205,6 +205,8 @@ export class RequestyHandler extends BaseProvider implements SingleCompletionHan metadata?.systemProviderOptions, ) + applyCacheBreakpoints(aiSdkMessages) + const requestOptions: Parameters[0] = { model: languageModel, system: effectiveSystemPrompt, diff --git a/src/api/providers/roo.ts b/src/api/providers/roo.ts index e426213a62..96a4d31373 100644 --- a/src/api/providers/roo.ts +++ b/src/api/providers/roo.ts @@ -17,7 +17,7 @@ import { mapToolChoice, yieldResponseMessage, } from "../transform/ai-sdk" -import { applyToolCacheOptions } from "../transform/cache-breakpoints" +import { applyCacheBreakpoints, applyToolCacheOptions } from "../transform/cache-breakpoints" import type { RooReasoningParams } from "../transform/reasoning" import { getRooReasoning } from "../transform/reasoning" @@ -161,6 +161,8 @@ export class RooHandler extends BaseProvider implements SingleCompletionHandler const tools = convertToolsForAiSdk(this.convertToolsForOpenAI(metadata?.tools)) applyToolCacheOptions(tools as Parameters[0], metadata?.toolProviderOptions) + applyCacheBreakpoints(aiSdkMessages) + let lastStreamError: string | undefined try { diff --git a/src/api/transform/__tests__/cache-breakpoints.spec.ts b/src/api/transform/__tests__/cache-breakpoints.spec.ts index c1b6c20701..81e8cd5ab8 100644 --- a/src/api/transform/__tests__/cache-breakpoints.spec.ts +++ b/src/api/transform/__tests__/cache-breakpoints.spec.ts @@ -44,21 +44,28 @@ describe("applyCacheBreakpoints", () => { expect(messages[0].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) - it("places 2 breakpoints on 2 user messages", () => { + it("places 1 breakpoint on the last of 2 user messages (default maxBreakpoints=1)", () => { const messages: TestMessage[] = [{ role: "user" }, { role: "user" }] applyCacheBreakpoints(messages) + expect(messages[0].providerOptions).toBeUndefined() + expect(messages[1].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + }) + + it("places 2 breakpoints on 2 user messages when maxBreakpoints=2", () => { + const messages: TestMessage[] = [{ role: "user" }, { role: "user" }] + applyCacheBreakpoints(messages, { maxBreakpoints: 2 }) expect(messages[0].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) expect(messages[1].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) - it("places 2 breakpoints on 2 tool messages", () => { + it("places 1 breakpoint on the last of 2 tool messages (default maxBreakpoints=1)", () => { const messages: TestMessage[] = [{ role: "tool" }, { role: "tool" }] applyCacheBreakpoints(messages) - expect(messages[0].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + expect(messages[0].providerOptions).toBeUndefined() expect(messages[1].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) - it("targets last 2 non-assistant messages in a mixed conversation", () => { + it("targets last non-assistant message in a mixed conversation", () => { const messages: TestMessage[] = [ { role: "user" }, { role: "assistant" }, @@ -67,15 +74,15 @@ describe("applyCacheBreakpoints", () => { { role: "tool" }, ] applyCacheBreakpoints(messages) - // Last 2 non-assistant: index 2 (user) and index 4 (tool) + // Last 1 non-assistant: index 4 (tool) expect(messages[0].providerOptions).toBeUndefined() expect(messages[1].providerOptions).toBeUndefined() - expect(messages[2].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + expect(messages[2].providerOptions).toBeUndefined() expect(messages[3].providerOptions).toBeUndefined() expect(messages[4].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) - it("targets indices 3 and 5 in [user, assistant, tool, user, assistant, tool]", () => { + it("targets only index 5 in [user, assistant, tool, user, assistant, tool]", () => { const messages: TestMessage[] = [ { role: "user" }, { role: "assistant" }, @@ -88,7 +95,7 @@ describe("applyCacheBreakpoints", () => { expect(messages[0].providerOptions).toBeUndefined() expect(messages[1].providerOptions).toBeUndefined() expect(messages[2].providerOptions).toBeUndefined() - expect(messages[3].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + expect(messages[3].providerOptions).toBeUndefined() expect(messages[4].providerOptions).toBeUndefined() expect(messages[5].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) @@ -97,7 +104,7 @@ describe("applyCacheBreakpoints", () => { const messages: TestMessage[] = [{ role: "system" }, { role: "user" }, { role: "assistant" }, { role: "user" }] applyCacheBreakpoints(messages) expect(messages[0].providerOptions).toBeUndefined() - expect(messages[1].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + expect(messages[1].providerOptions).toBeUndefined() expect(messages[3].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) @@ -132,7 +139,7 @@ describe("applyCacheBreakpoints", () => { it("adds anchor breakpoint at ~1/3 with useAnchor and enough messages", () => { // 6 non-assistant messages (indices 0-5 in nonAssistantIndices) // Anchor at floor(6/3) = index 2 in nonAssistantIndices -> messages index 4 - // Last 2: indices 10 and 8 + // Last 1: index 10 const messages: TestMessage[] = [ { role: "user" }, // 0 - nonAssistant[0] { role: "assistant" }, // 1 @@ -142,21 +149,21 @@ describe("applyCacheBreakpoints", () => { { role: "assistant" }, // 5 { role: "user" }, // 6 - nonAssistant[3] { role: "assistant" }, // 7 - { role: "user" }, // 8 - nonAssistant[4] <- last 2 + { role: "user" }, // 8 - nonAssistant[4] { role: "assistant" }, // 9 - { role: "user" }, // 10 - nonAssistant[5] <- last 2 + { role: "user" }, // 10 - nonAssistant[5] <- last 1 ] applyCacheBreakpoints(messages, { useAnchor: true }) - // Should have 3 breakpoints: indices 4, 8, 10 + // Should have 2 breakpoints: indices 4 (anchor) and 10 (last 1) expect(messages[4].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) - expect(messages[8].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) expect(messages[10].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) // Others should NOT have breakpoints expect(messages[0].providerOptions).toBeUndefined() expect(messages[2].providerOptions).toBeUndefined() expect(messages[6].providerOptions).toBeUndefined() + expect(messages[8].providerOptions).toBeUndefined() }) it("does not add anchor when below anchorThreshold", () => { @@ -170,9 +177,9 @@ describe("applyCacheBreakpoints", () => { // 3 non-assistant messages, below default threshold of 5 applyCacheBreakpoints(messages, { useAnchor: true }) - // Last 2 only: indices 2 and 4 + // Last 1 only: index 4 expect(messages[0].providerOptions).toBeUndefined() - expect(messages[2].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) + expect(messages[2].providerOptions).toBeUndefined() expect(messages[4].providerOptions).toEqual(UNIVERSAL_CACHE_OPTIONS) }) diff --git a/src/api/transform/cache-breakpoints.ts b/src/api/transform/cache-breakpoints.ts index 9bb912f3f5..8e9a713093 100644 --- a/src/api/transform/cache-breakpoints.ts +++ b/src/api/transform/cache-breakpoints.ts @@ -12,7 +12,7 @@ export const UNIVERSAL_CACHE_OPTIONS: Record> = * Optional targeting configuration for cache breakpoint placement. */ export interface CacheBreakpointTargeting { - /** Maximum number of message breakpoints to place. Default: 2 */ + /** Maximum number of message breakpoints to place. Default: 1 */ maxBreakpoints?: number /** Whether to add an anchor breakpoint at ~1/3 through the conversation. Default: false */ useAnchor?: boolean @@ -23,19 +23,19 @@ export interface CacheBreakpointTargeting { /** * Apply cache breakpoints to AI SDK messages with ALL provider namespaces. * - * 4-breakpoint strategy: + * 3-breakpoint strategy: * 1. System prompt — passed as first message in messages[] with providerOptions * 2. Tool definitions — handled externally via `toolProviderOptions` in `streamText()` - * 3-4. Last 2 non-assistant messages — this function handles these + * 3. Last non-assistant message — this function handles this * * @param messages - The AI SDK message array (mutated in place) - * @param targeting - Optional targeting options (defaults: 2 breakpoints, no anchor) + * @param targeting - Optional targeting options (defaults: 1 breakpoint, no anchor) */ export function applyCacheBreakpoints( messages: { role: string; providerOptions?: Record> }[], targeting: CacheBreakpointTargeting = {}, ): void { - const { maxBreakpoints = 2, useAnchor = false, anchorThreshold = 5 } = targeting + const { maxBreakpoints = 1, useAnchor = false, anchorThreshold = 5 } = targeting // 1. Collect non-assistant message indices (user | tool roles) const nonAssistantIndices: number[] = [] diff --git a/src/core/task-persistence/apiMessages.ts b/src/core/task-persistence/apiMessages.ts index 615d221875..36a2cf3330 100644 --- a/src/core/task-persistence/apiMessages.ts +++ b/src/core/task-persistence/apiMessages.ts @@ -282,10 +282,9 @@ export async function saveRooMessages({ try { const taskDir = await getTaskDirectoryPath(globalStoragePath, taskId) const filePath = path.join(taskDir, GlobalFileNames.apiConversationHistory) - const strippedMessages = stripCacheProviderOptions(messages) const envelope: RooMessageHistory = { version: ROO_MESSAGE_VERSION, - messages: strippedMessages, + messages, } await safeWriteJson(filePath, envelope) return true diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 8a91500db9..7e94208785 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -63,7 +63,7 @@ import { ApiHandler, ApiHandlerCreateMessageMetadata, buildApiHandler } from ".. import type { AssistantModelMessage } from "ai" import { ApiStream, GroundingSource } from "../../api/transform/stream" import { maybeRemoveImageBlocks } from "../../api/transform/image-cleaning" -import { applyCacheBreakpoints, UNIVERSAL_CACHE_OPTIONS } from "../../api/transform/cache-breakpoints" +import { UNIVERSAL_CACHE_OPTIONS } from "../../api/transform/cache-breakpoints" // shared import { findLastIndex } from "../../shared/array" @@ -4387,9 +4387,6 @@ export class Task extends EventEmitter implements TaskLike { const mergedForApi = mergeConsecutiveApiMessages(messagesSinceLastSummary, { roles: ["user"] }) const messagesWithoutImages = maybeRemoveImageBlocks(mergedForApi, this.api) - // Breakpoints 3-4: Apply cache breakpoints to the last 2 non-assistant messages - applyCacheBreakpoints(messagesWithoutImages.filter(isRooRoleMessage)) - // Check auto-approval limits const approvalResult = await this.autoApprovalHandler.checkAutoApprovalLimits( state,