diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 58fa931c74d..a1ff3840603 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -19,7 +19,7 @@ from litellm.llms.openai.openai import AllMessageValues from litellm.utils import is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url -from ..vertex_llm_base import VertexBase +from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents from .transformation import ( cached_messages_end_on_supported_turn, @@ -78,6 +78,20 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None: return timeout +def _cache_project_and_location( + custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], + api_base: str | None, + vertex_project: str | None, + vertex_location: str | None, +) -> tuple[str | None, str | None]: + model_location: Final = ( + None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base) + ) + if model_location is None: + return vertex_project, vertex_location + return model_location.project, model_location.location + + class ContextCachingEndpoints(VertexBase): """ Covers context caching endpoints for Vertex AI + Google AI Studio @@ -394,6 +408,9 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + cache_project, cache_location = _cache_project_and_location( + custom_llm_provider, api_base, vertex_project, vertex_location + ) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -402,8 +419,8 @@ class ContextCachingEndpoints(VertexBase): gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -440,8 +457,8 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -454,8 +471,8 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, ) cached_content_request_body["tools"] = tools @@ -490,7 +507,7 @@ class ContextCachingEndpoints(VertexBase): _record_cache_creation_tokens(logging_obj, raw_response_cached) self._known_caches.remember( KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location ), generated_cache_key, raw_response_cached.get("name"), @@ -560,6 +577,9 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + cache_project, cache_location = _cache_project_and_location( + custom_llm_provider, api_base, vertex_project, vertex_location + ) tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -568,8 +588,8 @@ class ContextCachingEndpoints(VertexBase): gemini_api_key=api_key, custom_llm_provider=custom_llm_provider, api_base=api_base, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -594,7 +614,7 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) scope: Final = KnownCachedContents.scope( - url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location + url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location ) async def lookup_or_create() -> str | None: @@ -606,8 +626,8 @@ class ContextCachingEndpoints(VertexBase): api_base=api_base, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, vertex_auth_header=vertex_auth_header, model=model, ) @@ -621,8 +641,8 @@ class ContextCachingEndpoints(VertexBase): messages=cached_messages, cache_key=generated_cache_key, custom_llm_provider=custom_llm_provider, - vertex_project=vertex_project, - vertex_location=vertex_location, + vertex_project=cache_project, + vertex_location=cache_location, ) cached_content_request_body["tools"] = tools diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 9a8b49bed50..82069e8d87f 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -10,6 +10,7 @@ import re import threading from collections import OrderedDict from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse @@ -48,14 +49,22 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str: _graft_default_vertex_path = graft_default_vertex_path -_VERTEX_LOCATION_PARENT: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+") +_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)") -def _vertex_collection_url(api_base: str, endpoint: str, default_url: str) -> str: - location_parent: Final = _VERTEX_LOCATION_PARENT.match(api_base) - if location_parent is None: - return join_api_base(api_base, urlparse(default_url).path) - return join_api_base(location_parent.group(), endpoint) +@dataclass(frozen=True, slots=True) +class PublisherModelLocation: + parent_url: str + project: str + location: str + + +def publisher_model_location(api_base: str) -> PublisherModelLocation | None: + match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base) + if match is None: + return None + _, _, project, _, location = match.group().rsplit("/", 4) + return PublisherModelLocation(parent_url=match.group(), project=project, location=location) GOOGLE_IMPORT_ERROR_MESSAGE: Final = ( @@ -659,9 +668,8 @@ class VertexBase: for a collection endpoint 2. Vertex AI with standard proxies - grafts the default vertex URL path onto the api_base when its path is empty or only an API version (/v1, /v1beta1); - otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint, the collection - under the api_base's projects/{project}/locations/{location} parent, or the default - collection path under an api_base with no location in it + otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base + naming a publisher model, the collection under that model's projects/{project}/locations/{location} 3. Vertex AI with PSC endpoints - constructs full path structure {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} (only when use_psc_endpoint_format=True) @@ -708,10 +716,11 @@ class VertexBase: elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: + model_location: Final = publisher_model_location(api_base) if collection_endpoint else None url = ( - _vertex_collection_url(api_base=api_base, endpoint=endpoint, default_url=url) - if collection_endpoint - else f"{api_base.rstrip('/')}:{endpoint}" + f"{api_base.rstrip('/')}:{endpoint}" + if model_location is None + else join_api_base(model_location.parent_url, endpoint) ) if stream is True: parsed_stream_url: Final = urlparse(url) diff --git a/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py b/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py index af8d0e902bd..b76f0cd162b 100644 --- a/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py +++ b/tests/integration/providers/test_gemini_context_caching_custom_api_base_wire.py @@ -12,8 +12,9 @@ from pydantic import JsonValue, TypeAdapter _BACKEND: Final = "gemini-3.8-flash" _PROJECT: Final = "scripted-project" _LOCATION: Final = "us-central1" -_VERTEX_LOCATION_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}" -_VERTEX_COLLECTION: Final = f"{_VERTEX_LOCATION_PATH}/cachedContents" +_MODEL_LOCATION: Final = "europe-west4" +_VERTEX_MODEL_RESOURCE: Final = f"projects/{_PROJECT}/locations/{_MODEL_LOCATION}/publishers/google/models/{_BACKEND}" +_VERTEX_COLLECTION: Final = f"/v1/projects/{_PROJECT}/locations/{_MODEL_LOCATION}/cachedContents" _GEMINI_COLLECTION: Final = "/v1beta/cachedContents" _GEMINI_MODEL_PATH: Final = f"/v1beta/models/{_BACKEND}" _GEMINI_KEY: Final = "scripted-gemini-key" @@ -105,18 +106,10 @@ def test_gemini_custom_api_base_lists_and_creates_the_cached_contents_collection assert _cached_tokens_of(response) == _CACHED_TOKENS -@pytest.mark.parametrize( - ("api_base_path", "collection"), - [ - (f"{_VERTEX_LOCATION_PATH}/publishers/google/models/{_BACKEND}", _VERTEX_COLLECTION), - (f"{_VERTEX_LOCATION_PATH}/endpoints/4417", _VERTEX_COLLECTION), - ("/vertex-proxy", f"/vertex-proxy{_VERTEX_COLLECTION}"), - ], - ids=["model-path", "endpoint-path", "mount-prefix"], -) -def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection( - gateway: Gateway, api_base_path: str, collection: str -) -> None: +@pytest.mark.parametrize("gateway_prefix", ["", "/vertex-proxy"], ids=["model-path", "prefixed-model-path"]) +def test_vertex_custom_api_base_creates_the_cache_where_its_model_lives(gateway: Gateway, gateway_prefix: str) -> None: + api_base_path: Final = f"{gateway_prefix}/v1/{_VERTEX_MODEL_RESOURCE}" + collection: Final = f"{gateway_prefix}{_VERTEX_COLLECTION}" with ( wire_server(_google_responder(collection, api_base_path)) as wire, gateway.scenario() as scenario, @@ -132,5 +125,6 @@ def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection response: Final = _chat(gateway, model) requests: Final = wire.drain() _assert_cache_round_trip(requests, collection, api_base_path) + assert _JSON_OBJECT.validate_json(requests[1].body)["model"] == _VERTEX_MODEL_RESOURCE assert all(request.headers.get("authorization") == "Bearer scripted-token" for request in requests) assert _cached_tokens_of(response) == _CACHED_TOKENS diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index bd1aa33f450..d665d5d6b75 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.common_utils import VertexAIError +from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCachedContents from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) @@ -224,6 +225,48 @@ class TestContextCachingEndpoints: model="gemini-1.5-pro", ) + @pytest.mark.parametrize("vertex_location", ["us-central1", None]) + def test_check_and_create_cache_creates_the_cache_where_the_api_base_model_lives( + self, vertex_location: str | None + ): + context_caching = ContextCachingEndpoints(known_caches=KnownCachedContents()) + listed = MagicMock() + listed.json.return_value = {"cachedContents": []} + created = MagicMock() + created.json.return_value = {"name": "cachedContents/base-cache"} + self.mock_client.get.return_value = listed + self.mock_client.post.return_value = created + base_model_location = "https://gateway.example.com/v1/projects/base-project/locations/europe-west4" + + _, _, cache_name = context_caching.check_and_create_cache( + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Long policy text", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "Summarize it"}, + ], + optional_params={}, + api_key="unused", + api_base=f"{base_model_location}/publishers/google/models/gemini-2.5-pro", + model="gemini-2.5-pro", + client=self.mock_client, + timeout=30.0, + logging_obj=self.mock_logging, + custom_llm_provider="vertex_ai", + vertex_project="config-project", + vertex_location=vertex_location, + vertex_auth_header="vertex-token", + ) + + assert cache_name == "cachedContents/base-cache" + assert self.mock_client.get.call_args.kwargs["url"] == f"{base_model_location}/cachedContents" + assert self.mock_client.post.call_args.kwargs["url"] == f"{base_model_location}/cachedContents" + assert ( + self.mock_client.post.call_args.kwargs["json"]["model"] + == "projects/base-project/locations/europe-west4/publishers/google/models/gemini-2.5-pro" + ) + @pytest.mark.parametrize( "custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"] ) @@ -2249,34 +2292,18 @@ class TestVertexAIGlobalLocation: "/publishers/google/models/gemini-2.5-pro", "https://gateway.example.com/v1/projects/my-project/locations/us-central1/cachedContents", ), - ( - "vertex_ai", - "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123", - "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/cachedContents", - ), ( "vertex_ai", "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" - "/publishers/google/models/gemini-2.5-pro", - "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" - "/cachedContents", - ), - ( - "vertex_ai", - "https://gateway.example.com/vertex-proxy", - "https://gateway.example.com/vertex-proxy/v1/projects/my-project/locations/us-central1/cachedContents", - ), - ( - "vertex_ai", - "https://gateway.example.com/publishers/relay", + "/publishers/google/models/gemini-2.5-pro/", "https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1" "/cachedContents", ), ( "vertex_ai_beta", - "https://gateway.example.com/vertex-proxy/", - "https://gateway.example.com/vertex-proxy/v1beta1/projects/my-project/locations/us-central1" - "/cachedContents", + "https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1" + "/publishers/google/models/gemini-2.5-pro", + "https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1/cachedContents", ), ], ) @@ -2298,6 +2325,31 @@ class TestVertexAIGlobalLocation: assert auth_header == "vertex-token" assert url == expected_url + @pytest.mark.parametrize( + "api_base", + [ + "https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123", + "https://gateway.example.com/v1/projects/my-project/locations/us-central1", + "https://gateway.example.com/vertex-proxy", + ], + ) + def test_vertex_context_caching_without_a_publisher_model_in_api_base_keeps_the_api_base_resource( + self, api_base: str + ): + caching = ContextCachingEndpoints() + + _, url = caching._get_token_and_url_context_caching( + gemini_api_key=None, + custom_llm_provider="vertex_ai", + api_base=api_base, + vertex_project="my-project", + vertex_location="us-central1", + vertex_auth_header="vertex-token", + model="gemini-2.5-pro", + ) + + assert url == f"{api_base}:cachedContents" + def test_gemini_context_caching_without_api_base_ignores_model(self): """Without custom api_base, model param is not needed (default URL is used).""" caching = ContextCachingEndpoints()