From f592e7da5488184b98e384b96cbe31f96dcb67b2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:23:34 -0700 Subject: [PATCH] fix(vertex_ai): cache under a publisher model api_base in that model's location A Vertex AI api_base that names an endpoint or no model keeps its old cache URL, so no cache is created that the chat call cannot use. The cache body takes its project and location from the api_base model, matching the collection URL. --- .../vertex_ai_context_caching.py | 50 +++++++--- litellm/llms/vertex_ai/vertex_llm_base.py | 33 ++++--- ...ni_context_caching_custom_api_base_wire.py | 22 ++--- .../test_vertex_ai_context_caching.py | 92 +++++++++++++++---- 4 files changed, 136 insertions(+), 61 deletions(-) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 58fa931c74d..a1ff3840603 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -19,7 +19,7 @@ from litellm.llms.openai.openai import AllMessageValues from litellm.utils import is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -78,6 +78,20 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +def _cache_project_and_location( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + vertex_project: str | None, + vertex_location: str | None, +) -> tuple[str | None, str | None]: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return vertex_project, vertex_location + return model_location.project, model_location.location + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -394,6 +408,9 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + cache_project, cache_location = _cache_project_and_location( + custom_llm_provider, api_base, vertex_project, vertex_location + ) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -402,8 +419,8 @@ class ContextCachingEndpoints(VertexBase): gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -440,8 +457,8 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -454,8 +471,8 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, ) cached_content_request_body["tools"] = tools @@ -490,7 +507,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location ), generated_cache_key, raw_response_cached.get("name"), @@ -560,6 +577,9 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + cache_project, cache_location = _cache_project_and_location( + custom_llm_provider, api_base, vertex_project, vertex_location + ) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -568,8 +588,8 @@ class ContextCachingEndpoints(VertexBase): gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -594,7 +614,7 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location ) async def lookup_or_create() -> str | None: @@ -606,8 +626,8 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -621,8 +641,8 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, ) cached_content_request_body["tools"] = tools diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 9a8b49bed50..82069e8d87f 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -10,6 +10,7 @@ import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -48,14 +49,22 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path -_VERTEX_LOCATION_PARENT: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+") +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") -def _vertex_collection_url(api_base: str, endpoint: str, default_url: str) -> str: - location_parent: Final = _VERTEX_LOCATION_PARENT.match(api_base) - if location_parent is None: - return join_api_base(api_base, urlparse(default_url).path) - return join_api_base(location_parent.group(), endpoint) +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation(parent_url=match.group(), project=project, location=location) GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( @@ -659,9 +668,8 @@ class VertexBase: for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint, the collection - under the api_base's projects/{project}/locations/{location} parent, or the default - collection path under an api_base with no location in it + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -708,10 +716,11 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None url = ( - _vertex_collection_url(api_base=api_base, endpoint=endpoint, default_url=url) - if collection_endpoint - else f"{api_base.rstrip('/')}:{endpoint}" + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) ) if stream is True: parsed_stream_url: Final = urlparse(url) diff --git a/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py b/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py index af8d0e902bd..b76f0cd162b 100644 --- a/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py +++ b/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py @@ -12,8 +12,9 @@ from pydantic import JsonValue, TypeAdapter _BACKEND: Final = "gemini-3.8-flash" _PROJECT: Final = "scripted-project" _LOCATION: Final = "us-central1" -_VERTEX_LOCATION_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}" -_VERTEX_COLLECTION: Final = f"{_VERTEX_LOCATION_PATH}/cachedContents" +_MODEL_LOCATION: Final = "europe-west4" +_VERTEX_MODEL_RESOURCE: Final = f"projects/{_PROJECT}/locations/{_MODEL_LOCATION}/publishers/google/models/{_BACKEND}" +_VERTEX_COLLECTION: Final = f"/v1/projects/{_PROJECT}/locations/{_MODEL_LOCATION}/cachedContents" _GEMINI_COLLECTION: Final = "/v1beta/cachedContents" _GEMINI_MODEL_PATH: Final = f"/v1beta/models/{_BACKEND}" _GEMINI_KEY: Final = "scripted-gemini-key" @@ -105,18 +106,10 @@ def test_gemini_custom_api_base_lists_and_creates_the_cached_contents_collection assert _cached_tokens_of(response) == _CACHED_TOKENS -@pytest.mark.parametrize( - ("api_base_path", "collection"), - [ - (f"{_VERTEX_LOCATION_PATH}/publishers/google/models/{_BACKEND}", _VERTEX_COLLECTION), - (f"{_VERTEX_LOCATION_PATH}/endpoints/4417", _VERTEX_COLLECTION), - ("/vertex-proxy", f"/vertex-proxy{_VERTEX_COLLECTION}"), - ], - ids=["model-path", "endpoint-path", "mount-prefix"], -) -def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection( - gateway: Gateway, api_base_path: str, collection: str -) -> None: +@pytest.mark.parametrize("gateway_prefix", ["", "/vertex-proxy"], ids=["model-path", "prefixed-model-path"]) +def test_vertex_custom_api_base_creates_the_cache_where_its_model_lives(gateway: Gateway, gateway_prefix: str) -> None: + api_base_path: Final = f"{gateway_prefix}/v1/{_VERTEX_MODEL_RESOURCE}" + collection: Final = f"{gateway_prefix}{_VERTEX_COLLECTION}" with ( wire_server(_google_responder(collection, api_base_path)) as wire, gateway.scenario() as scenario, @@ -132,5 +125,6 @@ def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection response: Final = _chat(gateway, model) requests: Final = wire.drain() _assert_cache_round_trip(requests, collection, api_base_path) + assert _JSON_OBJECT.validate_json(requests[1].body)["model"] == _VERTEX_MODEL_RESOURCE assert all(request.headers.get("authorization") == "Bearer scripted-token" for request in requests) assert _cached_tokens_of(response) == _CACHED_TOKENS diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index bd1aa33f450..d665d5d6b75 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.common_utils import VertexAIError +from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCachedContents from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) @@ -224,6 +225,48 @@ class TestContextCachingEndpoints: model="gemini-1.5-pro", ) + @pytest.mark.parametrize("vertex_location", ["us-central1", None]) + def test_check_and_create_cache_creates_the_cache_where_the_api_base_model_lives( + self, vertex_location: str | None + ): + context_caching = ContextCachingEndpoints(known_caches=KnownCachedContents()) + listed = MagicMock() + listed.json.return_value = {"cachedContents": []} + created = MagicMock() + created.json.return_value = {"name": "cachedContents/base-cache"} + self.mock_client.get.return_value = listed + self.mock_client.post.return_value = created + base_model_location = "https://gateway.example.com/v1/projects/base-project/locations/europe-west4" + + _, _, cache_name = context_caching.check_and_create_cache( + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Long policy text", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "Summarize it"}, + ], + optional_params={}, + api_key="unused", + api_base=f"{base_model_location}/publishers/google/models/gemini-2.5-pro", + model="gemini-2.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + custom_llm_provider="vertex_ai", + vertex_project="config-project", + vertex_location=vertex_location, + vertex_auth_header="vertex-token", + ) + + assert cache_name == "cachedContents/base-cache" + assert self.mock_client.get.call_args.kwargs["url"] == f"{base_model_location}/cachedContents" + assert self.mock_client.post.call_args.kwargs["url"] == f"{base_model_location}/cachedContents" + assert ( + self.mock_client.post.call_args.kwargs["json"]["model"] + == "projects/base-project/locations/europe-west4/publishers/google/models/gemini-2.5-pro" + ) + @pytest.mark.parametrize( "custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"] ) @@ -2249,34 +2292,18 @@ class TestVertexAIGlobalLocation: "/publishers/google/models/gemini-2.5-pro", "https://gateway.example.com/v1/projects/my-project/locations/us-central1/cachedContents", ), - ( - "vertex_ai", - "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123", - "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/cachedContents", - ), ( "vertex_ai", "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" - "/publishers/google/models/gemini-2.5-pro", - "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" - "/cachedContents", - ), - ( - "vertex_ai", - "https://gateway.example.com/vertex-proxy", - "https://gateway.example.com/vertex-proxy/v1/projects/my-project/locations/us-central1/cachedContents", - ), - ( - "vertex_ai", - "https://gateway.example.com/publishers/relay", + "/publishers/google/models/gemini-2.5-pro/", "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" "/cachedContents", ), ( "vertex_ai_beta", - "https://gateway.example.com/vertex-proxy/", - "https://gateway.example.com/vertex-proxy/v1beta1/projects/my-project/locations/us-central1" - "/cachedContents", + "https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1" + "/publishers/google/models/gemini-2.5-pro", + "https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1/cachedContents", ), ], ) @@ -2298,6 +2325,31 @@ class TestVertexAIGlobalLocation: assert auth_header == "vertex-token" assert url == expected_url + @pytest.mark.parametrize( + "api_base", + [ + "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123", + "https://gateway.example.com/v1/projects/my-project/locations/us-central1", + "https://gateway.example.com/vertex-proxy", + ], + ) + def test_vertex_context_caching_without_a_publisher_model_in_api_base_keeps_the_api_base_resource( + self, api_base: str + ): + caching = ContextCachingEndpoints() + + _, url = caching._get_token_and_url_context_caching( + gemini_api_key=None, + custom_llm_provider="vertex_ai", + api_base=api_base, + vertex_project="my-project", + vertex_location="us-central1", + vertex_auth_header="vertex-token", + model="gemini-2.5-pro", + ) + + assert url == f"{api_base}:cachedContents" + def test_gemini_context_caching_without_api_base_ignores_model(self): """Without custom api_base, model param is not needed (default URL is used).""" caching = ContextCachingEndpoints()