From 7ee90f916fc6579be271826da4609d55b1081af0 Mon Sep 17 00:00:00 2001 From: pugazhendhi-m <132246623+pugazhendhi-m@users.noreply.github.com> Date: Fri, 30 May 2025 12:03:12 +0530 Subject: [PATCH] Refactor model cache breakpoint logic to use model info in Unbound (#4137) Co-authored-by: Pugazhendhi --- src/api/providers/unbound.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/api/providers/unbound.ts b/src/api/providers/unbound.ts index 5ca38a8514..20ec1b54bc 100644 --- a/src/api/providers/unbound.ts +++ b/src/api/providers/unbound.ts @@ -5,7 +5,8 @@ import { ApiHandlerOptions, unboundDefaultModelId, unboundDefaultModelInfo } fro import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" -import { addCacheBreakpoints } from "../transform/caching/anthropic" +import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" +import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" import { RouterProvider } from "./router-provider" @@ -44,8 +45,12 @@ export class UnboundHandler extends RouterProvider implements SingleCompletionHa ...convertToOpenAiMessages(messages), ] - if (modelId.startsWith("anthropic/claude-3")) { - addCacheBreakpoints(systemPrompt, openAiMessages) + if (info.supportsPromptCache) { + if (modelId.startsWith("google/")) { + addGeminiCacheBreakpoints(systemPrompt, openAiMessages) + } else if (modelId.startsWith("anthropic/")) { + addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) + } } // Required by Anthropic; other providers default to max tokens allowed.