fix(vertex_ai): cache under a publisher model api_base in that model's location

A Vertex AI api_base that names an endpoint or no model keeps its old cache URL, so
no cache is created that the chat call cannot use. The cache body takes its project and
location from the api_base model, matching the collection URL.
This commit is contained in:
mateo-berri 2026-10-28 18:23:34 -07:00
parent 41d9ad5f47
commit f592e7da54
4 changed files with 136 additions and 61 deletions

View file

@ -19,7 +19,7 @@ from litellm.llms.openai.openai import AllMessageValues
from litellm.utils import is_prompt_caching_valid_prompt
from ..common_utils import VertexAIError, get_vertex_base_url
from ..vertex_llm_base import VertexBase
from ..vertex_llm_base import VertexBase, publisher_model_location
from .known_cached_contents import KnownCachedContents, known_cached_contents
from .transformation import (
cached_messages_end_on_supported_turn,
@ -78,6 +78,20 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None:
return timeout
def _cache_project_and_location(
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
api_base: str | None,
vertex_project: str | None,
vertex_location: str | None,
) -> tuple[str | None, str | None]:
model_location: Final = (
None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base)
)
if model_location is None:
return vertex_project, vertex_location
return model_location.project, model_location.location
class ContextCachingEndpoints(VertexBase):
"""
Covers context caching endpoints for Vertex AI + Google AI Studio
@ -394,6 +408,9 @@ class ContextCachingEndpoints(VertexBase):
)
return messages, optional_params, None
cache_project, cache_location = _cache_project_and_location(
custom_llm_provider, api_base, vertex_project, vertex_location
)
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)
@ -402,8 +419,8 @@ class ContextCachingEndpoints(VertexBase):
gemini_api_key=api_key,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
vertex_auth_header=vertex_auth_header,
model=model,
)
@ -440,8 +457,8 @@ class ContextCachingEndpoints(VertexBase):
api_base=api_base,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
vertex_auth_header=vertex_auth_header,
model=model,
)
@ -454,8 +471,8 @@ class ContextCachingEndpoints(VertexBase):
messages=cached_messages,
cache_key=generated_cache_key,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
)
cached_content_request_body["tools"] = tools
@ -490,7 +507,7 @@ class ContextCachingEndpoints(VertexBase):
_record_cache_creation_tokens(logging_obj, raw_response_cached)
self._known_caches.remember(
KnownCachedContents.scope(
url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location
),
generated_cache_key,
raw_response_cached.get("name"),
@ -560,6 +577,9 @@ class ContextCachingEndpoints(VertexBase):
)
return messages, optional_params, None
cache_project, cache_location = _cache_project_and_location(
custom_llm_provider, api_base, vertex_project, vertex_location
)
tools: Final = optional_params.pop("tools", None)
tool_choice: Final = optional_params.pop("tool_choice", None)
@ -568,8 +588,8 @@ class ContextCachingEndpoints(VertexBase):
gemini_api_key=api_key,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
vertex_auth_header=vertex_auth_header,
model=model,
)
@ -594,7 +614,7 @@ class ContextCachingEndpoints(VertexBase):
messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model
)
scope: Final = KnownCachedContents.scope(
url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location
)
async def lookup_or_create() -> str | None:
@ -606,8 +626,8 @@ class ContextCachingEndpoints(VertexBase):
api_base=api_base,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
vertex_auth_header=vertex_auth_header,
model=model,
)
@ -621,8 +641,8 @@ class ContextCachingEndpoints(VertexBase):
messages=cached_messages,
cache_key=generated_cache_key,
custom_llm_provider=custom_llm_provider,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_project=cache_project,
vertex_location=cache_location,
)
cached_content_request_body["tools"] = tools

View file

@ -10,6 +10,7 @@ import re
import threading
from collections import OrderedDict
from collections.abc import Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
from urllib.parse import urlparse
@ -48,14 +49,22 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str:
_graft_default_vertex_path = graft_default_vertex_path
_VERTEX_LOCATION_PARENT: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+")
_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)")
def _vertex_collection_url(api_base: str, endpoint: str, default_url: str) -> str:
location_parent: Final = _VERTEX_LOCATION_PARENT.match(api_base)
if location_parent is None:
return join_api_base(api_base, urlparse(default_url).path)
return join_api_base(location_parent.group(), endpoint)
@dataclass(frozen=True, slots=True)
class PublisherModelLocation:
parent_url: str
project: str
location: str
def publisher_model_location(api_base: str) -> PublisherModelLocation | None:
match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base)
if match is None:
return None
_, _, project, _, location = match.group().rsplit("/", 4)
return PublisherModelLocation(parent_url=match.group(), project=project, location=location)
GOOGLE_IMPORT_ERROR_MESSAGE: Final = (
@ -659,9 +668,8 @@ class VertexBase:
for a collection endpoint
2. Vertex AI with standard proxies - grafts the default vertex URL path onto the
api_base when its path is empty or only an API version (/v1, /v1beta1);
otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint, the collection
under the api_base's projects/{project}/locations/{location} parent, or the default
collection path under an api_base with no location in it
otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base
naming a publisher model, the collection under that model's projects/{project}/locations/{location}
3. Vertex AI with PSC endpoints - constructs full path structure
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
(only when use_psc_endpoint_format=True)
@ -708,10 +716,11 @@ class VertexBase:
elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path:
url = graft_default_vertex_path(api_base=api_base, default_url=url)
else:
model_location: Final = publisher_model_location(api_base) if collection_endpoint else None
url = (
_vertex_collection_url(api_base=api_base, endpoint=endpoint, default_url=url)
if collection_endpoint
else f"{api_base.rstrip('/')}:{endpoint}"
f"{api_base.rstrip('/')}:{endpoint}"
if model_location is None
else join_api_base(model_location.parent_url, endpoint)
)
if stream is True:
parsed_stream_url: Final = urlparse(url)

View file

@ -12,8 +12,9 @@ from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "gemini-3.8-flash"
_PROJECT: Final = "scripted-project"
_LOCATION: Final = "us-central1"
_VERTEX_LOCATION_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}"
_VERTEX_COLLECTION: Final = f"{_VERTEX_LOCATION_PATH}/cachedContents"
_MODEL_LOCATION: Final = "europe-west4"
_VERTEX_MODEL_RESOURCE: Final = f"projects/{_PROJECT}/locations/{_MODEL_LOCATION}/publishers/google/models/{_BACKEND}"
_VERTEX_COLLECTION: Final = f"/v1/projects/{_PROJECT}/locations/{_MODEL_LOCATION}/cachedContents"
_GEMINI_COLLECTION: Final = "/v1beta/cachedContents"
_GEMINI_MODEL_PATH: Final = f"/v1beta/models/{_BACKEND}"
_GEMINI_KEY: Final = "scripted-gemini-key"
@ -105,18 +106,10 @@ def test_gemini_custom_api_base_lists_and_creates_the_cached_contents_collection
assert _cached_tokens_of(response) == _CACHED_TOKENS
@pytest.mark.parametrize(
("api_base_path", "collection"),
[
(f"{_VERTEX_LOCATION_PATH}/publishers/google/models/{_BACKEND}", _VERTEX_COLLECTION),
(f"{_VERTEX_LOCATION_PATH}/endpoints/4417", _VERTEX_COLLECTION),
("/vertex-proxy", f"/vertex-proxy{_VERTEX_COLLECTION}"),
],
ids=["model-path", "endpoint-path", "mount-prefix"],
)
def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection(
gateway: Gateway, api_base_path: str, collection: str
) -> None:
@pytest.mark.parametrize("gateway_prefix", ["", "/vertex-proxy"], ids=["model-path", "prefixed-model-path"])
def test_vertex_custom_api_base_creates_the_cache_where_its_model_lives(gateway: Gateway, gateway_prefix: str) -> None:
api_base_path: Final = f"{gateway_prefix}/v1/{_VERTEX_MODEL_RESOURCE}"
collection: Final = f"{gateway_prefix}{_VERTEX_COLLECTION}"
with (
wire_server(_google_responder(collection, api_base_path)) as wire,
gateway.scenario() as scenario,
@ -132,5 +125,6 @@ def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection
response: Final = _chat(gateway, model)
requests: Final = wire.drain()
_assert_cache_round_trip(requests, collection, api_base_path)
assert _JSON_OBJECT.validate_json(requests[1].body)["model"] == _VERTEX_MODEL_RESOURCE
assert all(request.headers.get("authorization") == "Bearer scripted-token" for request in requests)
assert _cached_tokens_of(response) == _CACHED_TOKENS

View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCachedContents
from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import (
ContextCachingEndpoints,
)
@ -224,6 +225,48 @@ class TestContextCachingEndpoints:
model="gemini-1.5-pro",
)
@pytest.mark.parametrize("vertex_location", ["us-central1", None])
def test_check_and_create_cache_creates_the_cache_where_the_api_base_model_lives(
self, vertex_location: str | None
):
context_caching = ContextCachingEndpoints(known_caches=KnownCachedContents())
listed = MagicMock()
listed.json.return_value = {"cachedContents": []}
created = MagicMock()
created.json.return_value = {"name": "cachedContents/base-cache"}
self.mock_client.get.return_value = listed
self.mock_client.post.return_value = created
base_model_location = "https://gateway.example.com/v1/projects/base-project/locations/europe-west4"
_, _, cache_name = context_caching.check_and_create_cache(
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "Long policy text", "cache_control": {"type": "ephemeral"}}],
},
{"role": "user", "content": "Summarize it"},
],
optional_params={},
api_key="unused",
api_base=f"{base_model_location}/publishers/google/models/gemini-2.5-pro",
model="gemini-2.5-pro",
client=self.mock_client,
timeout=30.0,
logging_obj=self.mock_logging,
custom_llm_provider="vertex_ai",
vertex_project="config-project",
vertex_location=vertex_location,
vertex_auth_header="vertex-token",
)
assert cache_name == "cachedContents/base-cache"
assert self.mock_client.get.call_args.kwargs["url"] == f"{base_model_location}/cachedContents"
assert self.mock_client.post.call_args.kwargs["url"] == f"{base_model_location}/cachedContents"
assert (
self.mock_client.post.call_args.kwargs["json"]["model"]
== "projects/base-project/locations/europe-west4/publishers/google/models/gemini-2.5-pro"
)
@pytest.mark.parametrize(
"custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]
)
@ -2249,34 +2292,18 @@ class TestVertexAIGlobalLocation:
"/publishers/google/models/gemini-2.5-pro",
"https://gateway.example.com/v1/projects/my-project/locations/us-central1/cachedContents",
),
(
"vertex_ai",
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123",
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/cachedContents",
),
(
"vertex_ai",
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
"/publishers/google/models/gemini-2.5-pro",
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
"/cachedContents",
),
(
"vertex_ai",
"https://gateway.example.com/vertex-proxy",
"https://gateway.example.com/vertex-proxy/v1/projects/my-project/locations/us-central1/cachedContents",
),
(
"vertex_ai",
"https://gateway.example.com/publishers/relay",
"/publishers/google/models/gemini-2.5-pro/",
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
"/cachedContents",
),
(
"vertex_ai_beta",
"https://gateway.example.com/vertex-proxy/",
"https://gateway.example.com/vertex-proxy/v1beta1/projects/my-project/locations/us-central1"
"/cachedContents",
"https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1"
"/publishers/google/models/gemini-2.5-pro",
"https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1/cachedContents",
),
],
)
@ -2298,6 +2325,31 @@ class TestVertexAIGlobalLocation:
assert auth_header == "vertex-token"
assert url == expected_url
@pytest.mark.parametrize(
"api_base",
[
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123",
"https://gateway.example.com/v1/projects/my-project/locations/us-central1",
"https://gateway.example.com/vertex-proxy",
],
)
def test_vertex_context_caching_without_a_publisher_model_in_api_base_keeps_the_api_base_resource(
self, api_base: str
):
caching = ContextCachingEndpoints()
_, url = caching._get_token_and_url_context_caching(
gemini_api_key=None,
custom_llm_provider="vertex_ai",
api_base=api_base,
vertex_project="my-project",
vertex_location="us-central1",
vertex_auth_header="vertex-token",
model="gemini-2.5-pro",
)
assert url == f"{api_base}:cachedContents"
def test_gemini_context_caching_without_api_base_ignores_model(self):
"""Without custom api_base, model param is not needed (default URL is used)."""
caching = ContextCachingEndpoints()