mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vertex_ai): cache under a publisher model api_base in that model's location
A Vertex AI api_base that names an endpoint or no model keeps its old cache URL, so no cache is created that the chat call cannot use. The cache body takes its project and location from the api_base model, matching the collection URL.
This commit is contained in:
parent
41d9ad5f47
commit
f592e7da54
4 changed files with 136 additions and 61 deletions
|
|
@ -19,7 +19,7 @@ from litellm.llms.openai.openai import AllMessageValues
|
|||
from litellm.utils import is_prompt_caching_valid_prompt
|
||||
|
||||
from ..common_utils import VertexAIError, get_vertex_base_url
|
||||
from ..vertex_llm_base import VertexBase
|
||||
from ..vertex_llm_base import VertexBase, publisher_model_location
|
||||
from .known_cached_contents import KnownCachedContents, known_cached_contents
|
||||
from .transformation import (
|
||||
cached_messages_end_on_supported_turn,
|
||||
|
|
@ -78,6 +78,20 @@ def _wait_seconds(timeout: float | httpx.Timeout | None) -> float | None:
|
|||
return timeout
|
||||
|
||||
|
||||
def _cache_project_and_location(
|
||||
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
|
||||
api_base: str | None,
|
||||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
model_location: Final = (
|
||||
None if custom_llm_provider == "gemini" or api_base is None else publisher_model_location(api_base)
|
||||
)
|
||||
if model_location is None:
|
||||
return vertex_project, vertex_location
|
||||
return model_location.project, model_location.location
|
||||
|
||||
|
||||
class ContextCachingEndpoints(VertexBase):
|
||||
"""
|
||||
Covers context caching endpoints for Vertex AI + Google AI Studio
|
||||
|
|
@ -394,6 +408,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
cache_project, cache_location = _cache_project_and_location(
|
||||
custom_llm_provider, api_base, vertex_project, vertex_location
|
||||
)
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
@ -402,8 +419,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
gemini_api_key=api_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
|
@ -440,8 +457,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
|
@ -454,8 +471,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
messages=cached_messages,
|
||||
cache_key=generated_cache_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
)
|
||||
|
||||
cached_content_request_body["tools"] = tools
|
||||
|
|
@ -490,7 +507,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
_record_cache_creation_tokens(logging_obj, raw_response_cached)
|
||||
self._known_caches.remember(
|
||||
KnownCachedContents.scope(
|
||||
url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
|
||||
url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location
|
||||
),
|
||||
generated_cache_key,
|
||||
raw_response_cached.get("name"),
|
||||
|
|
@ -560,6 +577,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
cache_project, cache_location = _cache_project_and_location(
|
||||
custom_llm_provider, api_base, vertex_project, vertex_location
|
||||
)
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
@ -568,8 +588,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
gemini_api_key=api_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
|
@ -594,7 +614,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model
|
||||
)
|
||||
scope: Final = KnownCachedContents.scope(
|
||||
url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
|
||||
url, custom_llm_provider, headers, vertex_project=cache_project, vertex_location=cache_location
|
||||
)
|
||||
|
||||
async def lookup_or_create() -> str | None:
|
||||
|
|
@ -606,8 +626,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
api_base=api_base,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
|
@ -621,8 +641,8 @@ class ContextCachingEndpoints(VertexBase):
|
|||
messages=cached_messages,
|
||||
cache_key=generated_cache_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_project=cache_project,
|
||||
vertex_location=cache_location,
|
||||
)
|
||||
|
||||
cached_content_request_body["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import re
|
|||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -48,14 +49,22 @@ def graft_default_vertex_path(api_base: str, default_url: str) -> str:
|
|||
_graft_default_vertex_path = graft_default_vertex_path
|
||||
|
||||
|
||||
_VERTEX_LOCATION_PARENT: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+")
|
||||
_PUBLISHER_MODEL_LOCATION: Final = re.compile(r".*/projects/[^/]+/locations/[^/]+(?=/publishers/[^/]+/models/[^/]+/?$)")
|
||||
|
||||
|
||||
def _vertex_collection_url(api_base: str, endpoint: str, default_url: str) -> str:
|
||||
location_parent: Final = _VERTEX_LOCATION_PARENT.match(api_base)
|
||||
if location_parent is None:
|
||||
return join_api_base(api_base, urlparse(default_url).path)
|
||||
return join_api_base(location_parent.group(), endpoint)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublisherModelLocation:
|
||||
parent_url: str
|
||||
project: str
|
||||
location: str
|
||||
|
||||
|
||||
def publisher_model_location(api_base: str) -> PublisherModelLocation | None:
|
||||
match: Final = _PUBLISHER_MODEL_LOCATION.match(api_base)
|
||||
if match is None:
|
||||
return None
|
||||
_, _, project, _, location = match.group().rsplit("/", 4)
|
||||
return PublisherModelLocation(parent_url=match.group(), project=project, location=location)
|
||||
|
||||
|
||||
GOOGLE_IMPORT_ERROR_MESSAGE: Final = (
|
||||
|
|
@ -659,9 +668,8 @@ class VertexBase:
|
|||
for a collection endpoint
|
||||
2. Vertex AI with standard proxies - grafts the default vertex URL path onto the
|
||||
api_base when its path is empty or only an API version (/v1, /v1beta1);
|
||||
otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint, the collection
|
||||
under the api_base's projects/{project}/locations/{location} parent, or the default
|
||||
collection path under an api_base with no location in it
|
||||
otherwise constructs {api_base}:{endpoint}, or, for a collection endpoint under an api_base
|
||||
naming a publisher model, the collection under that model's projects/{project}/locations/{location}
|
||||
3. Vertex AI with PSC endpoints - constructs full path structure
|
||||
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
|
||||
(only when use_psc_endpoint_format=True)
|
||||
|
|
@ -708,10 +716,11 @@ class VertexBase:
|
|||
elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path:
|
||||
url = graft_default_vertex_path(api_base=api_base, default_url=url)
|
||||
else:
|
||||
model_location: Final = publisher_model_location(api_base) if collection_endpoint else None
|
||||
url = (
|
||||
_vertex_collection_url(api_base=api_base, endpoint=endpoint, default_url=url)
|
||||
if collection_endpoint
|
||||
else f"{api_base.rstrip('/')}:{endpoint}"
|
||||
f"{api_base.rstrip('/')}:{endpoint}"
|
||||
if model_location is None
|
||||
else join_api_base(model_location.parent_url, endpoint)
|
||||
)
|
||||
if stream is True:
|
||||
parsed_stream_url: Final = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -12,8 +12,9 @@ from pydantic import JsonValue, TypeAdapter
|
|||
_BACKEND: Final = "gemini-3.8-flash"
|
||||
_PROJECT: Final = "scripted-project"
|
||||
_LOCATION: Final = "us-central1"
|
||||
_VERTEX_LOCATION_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}"
|
||||
_VERTEX_COLLECTION: Final = f"{_VERTEX_LOCATION_PATH}/cachedContents"
|
||||
_MODEL_LOCATION: Final = "europe-west4"
|
||||
_VERTEX_MODEL_RESOURCE: Final = f"projects/{_PROJECT}/locations/{_MODEL_LOCATION}/publishers/google/models/{_BACKEND}"
|
||||
_VERTEX_COLLECTION: Final = f"/v1/projects/{_PROJECT}/locations/{_MODEL_LOCATION}/cachedContents"
|
||||
_GEMINI_COLLECTION: Final = "/v1beta/cachedContents"
|
||||
_GEMINI_MODEL_PATH: Final = f"/v1beta/models/{_BACKEND}"
|
||||
_GEMINI_KEY: Final = "scripted-gemini-key"
|
||||
|
|
@ -105,18 +106,10 @@ def test_gemini_custom_api_base_lists_and_creates_the_cached_contents_collection
|
|||
assert _cached_tokens_of(response) == _CACHED_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_base_path", "collection"),
|
||||
[
|
||||
(f"{_VERTEX_LOCATION_PATH}/publishers/google/models/{_BACKEND}", _VERTEX_COLLECTION),
|
||||
(f"{_VERTEX_LOCATION_PATH}/endpoints/4417", _VERTEX_COLLECTION),
|
||||
("/vertex-proxy", f"/vertex-proxy{_VERTEX_COLLECTION}"),
|
||||
],
|
||||
ids=["model-path", "endpoint-path", "mount-prefix"],
|
||||
)
|
||||
def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection(
|
||||
gateway: Gateway, api_base_path: str, collection: str
|
||||
) -> None:
|
||||
@pytest.mark.parametrize("gateway_prefix", ["", "/vertex-proxy"], ids=["model-path", "prefixed-model-path"])
|
||||
def test_vertex_custom_api_base_creates_the_cache_where_its_model_lives(gateway: Gateway, gateway_prefix: str) -> None:
|
||||
api_base_path: Final = f"{gateway_prefix}/v1/{_VERTEX_MODEL_RESOURCE}"
|
||||
collection: Final = f"{gateway_prefix}{_VERTEX_COLLECTION}"
|
||||
with (
|
||||
wire_server(_google_responder(collection, api_base_path)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
|
|
@ -132,5 +125,6 @@ def test_vertex_custom_api_base_lists_and_creates_the_cached_contents_collection
|
|||
response: Final = _chat(gateway, model)
|
||||
requests: Final = wire.drain()
|
||||
_assert_cache_round_trip(requests, collection, api_base_path)
|
||||
assert _JSON_OBJECT.validate_json(requests[1].body)["model"] == _VERTEX_MODEL_RESOURCE
|
||||
assert all(request.headers.get("authorization") == "Bearer scripted-token" for request in requests)
|
||||
assert _cached_tokens_of(response) == _CACHED_TOKENS
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIError
|
||||
from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCachedContents
|
||||
from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import (
|
||||
ContextCachingEndpoints,
|
||||
)
|
||||
|
|
@ -224,6 +225,48 @@ class TestContextCachingEndpoints:
|
|||
model="gemini-1.5-pro",
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("vertex_location", ["us-central1", None])
|
||||
def test_check_and_create_cache_creates_the_cache_where_the_api_base_model_lives(
|
||||
self, vertex_location: str | None
|
||||
):
|
||||
context_caching = ContextCachingEndpoints(known_caches=KnownCachedContents())
|
||||
listed = MagicMock()
|
||||
listed.json.return_value = {"cachedContents": []}
|
||||
created = MagicMock()
|
||||
created.json.return_value = {"name": "cachedContents/base-cache"}
|
||||
self.mock_client.get.return_value = listed
|
||||
self.mock_client.post.return_value = created
|
||||
base_model_location = "https://gateway.example.com/v1/projects/base-project/locations/europe-west4"
|
||||
|
||||
_, _, cache_name = context_caching.check_and_create_cache(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Long policy text", "cache_control": {"type": "ephemeral"}}],
|
||||
},
|
||||
{"role": "user", "content": "Summarize it"},
|
||||
],
|
||||
optional_params={},
|
||||
api_key="unused",
|
||||
api_base=f"{base_model_location}/publishers/google/models/gemini-2.5-pro",
|
||||
model="gemini-2.5-pro",
|
||||
client=self.mock_client,
|
||||
timeout=30.0,
|
||||
logging_obj=self.mock_logging,
|
||||
custom_llm_provider="vertex_ai",
|
||||
vertex_project="config-project",
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header="vertex-token",
|
||||
)
|
||||
|
||||
assert cache_name == "cachedContents/base-cache"
|
||||
assert self.mock_client.get.call_args.kwargs["url"] == f"{base_model_location}/cachedContents"
|
||||
assert self.mock_client.post.call_args.kwargs["url"] == f"{base_model_location}/cachedContents"
|
||||
assert (
|
||||
self.mock_client.post.call_args.kwargs["json"]["model"]
|
||||
== "projects/base-project/locations/europe-west4/publishers/google/models/gemini-2.5-pro"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]
|
||||
)
|
||||
|
|
@ -2249,34 +2292,18 @@ class TestVertexAIGlobalLocation:
|
|||
"/publishers/google/models/gemini-2.5-pro",
|
||||
"https://gateway.example.com/v1/projects/my-project/locations/us-central1/cachedContents",
|
||||
),
|
||||
(
|
||||
"vertex_ai",
|
||||
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123",
|
||||
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/cachedContents",
|
||||
),
|
||||
(
|
||||
"vertex_ai",
|
||||
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
|
||||
"/publishers/google/models/gemini-2.5-pro",
|
||||
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
|
||||
"/cachedContents",
|
||||
),
|
||||
(
|
||||
"vertex_ai",
|
||||
"https://gateway.example.com/vertex-proxy",
|
||||
"https://gateway.example.com/vertex-proxy/v1/projects/my-project/locations/us-central1/cachedContents",
|
||||
),
|
||||
(
|
||||
"vertex_ai",
|
||||
"https://gateway.example.com/publishers/relay",
|
||||
"/publishers/google/models/gemini-2.5-pro/",
|
||||
"https://gateway.example.com/publishers/relay/v1/projects/my-project/locations/us-central1"
|
||||
"/cachedContents",
|
||||
),
|
||||
(
|
||||
"vertex_ai_beta",
|
||||
"https://gateway.example.com/vertex-proxy/",
|
||||
"https://gateway.example.com/vertex-proxy/v1beta1/projects/my-project/locations/us-central1"
|
||||
"/cachedContents",
|
||||
"https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1"
|
||||
"/publishers/google/models/gemini-2.5-pro",
|
||||
"https://gateway.example.com/v1beta1/projects/my-project/locations/us-central1/cachedContents",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
|
@ -2298,6 +2325,31 @@ class TestVertexAIGlobalLocation:
|
|||
assert auth_header == "vertex-token"
|
||||
assert url == expected_url
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[
|
||||
"https://gateway.example.com/v1/projects/tuned-project/locations/europe-west4/endpoints/123",
|
||||
"https://gateway.example.com/v1/projects/my-project/locations/us-central1",
|
||||
"https://gateway.example.com/vertex-proxy",
|
||||
],
|
||||
)
|
||||
def test_vertex_context_caching_without_a_publisher_model_in_api_base_keeps_the_api_base_resource(
|
||||
self, api_base: str
|
||||
):
|
||||
caching = ContextCachingEndpoints()
|
||||
|
||||
_, url = caching._get_token_and_url_context_caching(
|
||||
gemini_api_key=None,
|
||||
custom_llm_provider="vertex_ai",
|
||||
api_base=api_base,
|
||||
vertex_project="my-project",
|
||||
vertex_location="us-central1",
|
||||
vertex_auth_header="vertex-token",
|
||||
model="gemini-2.5-pro",
|
||||
)
|
||||
|
||||
assert url == f"{api_base}:cachedContents"
|
||||
|
||||
def test_gemini_context_caching_without_api_base_ignores_model(self):
|
||||
"""Without custom api_base, model param is not needed (default URL is used)."""
|
||||
caching = ContextCachingEndpoints()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue