From 2cb02d6512360248091cdb30353a4c6b9f525c0c Mon Sep 17 00:00:00 2001 From: pugazhendhi-m <132246623+pugazhendhi-m@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:59:40 +0530 Subject: [PATCH] Add cache breakpoints for custom vertex models on Unbound (#4585) Co-authored-by: Pugazhendhi --- src/api/providers/unbound.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/api/providers/unbound.ts b/src/api/providers/unbound.ts index 2c7bd1e575..bc85dfd499 100644 --- a/src/api/providers/unbound.ts +++ b/src/api/providers/unbound.ts @@ -9,6 +9,7 @@ import { ApiStream, ApiStreamUsageChunk } from "../transform/stream" import { convertToOpenAiMessages } from "../transform/openai-format" import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic" import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini" +import { addCacheBreakpoints as addVertexCacheBreakpoints } from "../transform/caching/vertex" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index" import { RouterProvider } from "./router-provider" @@ -70,6 +71,10 @@ export class UnboundHandler extends RouterProvider implements SingleCompletionHa addAnthropicCacheBreakpoints(systemPrompt, openAiMessages) } } + // Custom models from Vertex AI (no configuration) need to be handled differently. + if (modelId.startsWith("vertex-ai/google.") || modelId.startsWith("vertex-ai/anthropic.")) { + addVertexCacheBreakpoints(messages) + } // Required by Anthropic; other providers default to max tokens allowed. let maxTokens: number | undefined