From 5fe503c3266f893226bafa36b2f98ea483082693 Mon Sep 17 00:00:00 2001 From: scriptsmay Date: Tue, 3 Nov 2026 15:52:53 +0800 Subject: [PATCH 1/3] fix(model_prices): set supports_prompt_caching to false for gemini-4-argon models --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 376cd816c66..54dac756e9c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76859,7 +76859,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, @@ -76919,7 +76919,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, @@ -76977,7 +76977,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 376cd816c66..54dac756e9c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76859,7 +76859,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, @@ -76919,7 +76919,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, @@ -76977,7 +76977,7 @@ "supports_native_streaming": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, From 4218cc95eaf29909c1e990d0ae5c8ded5d9e64a2 Mon Sep 17 00:00:00 2001 From: scriptsmay Date: Tue, 3 Nov 2026 18:11:32 +0800 Subject: [PATCH 2/3] fix(vertex_ai): bypass context caching and fallback when explicit caching is unsupported (#46470) --- .../messages/adapters/transformation.py | 10 +- .../vertex_ai_context_caching.py | 30 ++++ .../llms/vertex_ai/gemini/transformation.py | 5 + ..._vertex_ai_explicit_caching_unsupported.py | 162 ++++++++++++++++++ 4 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b1b5a8a11a..4b027f636c9 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -735,7 +735,15 @@ class LiteLLMAnthropicMessagesAdapter: @classmethod def target_consumes_cache_control(cls, model: str) -> bool: - return cls.is_anthropic_claude_model(model) or cls.is_bedrock_arn_model(model) or "gemini" in model.lower() + if cls.is_anthropic_claude_model(model) or cls.is_bedrock_arn_model(model): + return True + if "gemini" in model.lower(): + from litellm.utils import is_explicitly_disabled_factory + + if is_explicitly_disabled_factory(model, None, "supports_prompt_caching"): + return False + return True + return False @staticmethod def translate_thinking_for_model( diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 4603f612926..b23f29340de 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -130,6 +130,14 @@ def _context_caching_skip_reason( cache_model: str, custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"], ) -> str | None: + from litellm.utils import is_explicitly_disabled_factory + + if is_explicitly_disabled_factory(deployment_model, custom_llm_provider, "supports_prompt_caching") or ( + cache_model != deployment_model + and is_explicitly_disabled_factory(cache_model, custom_llm_provider, "supports_prompt_caching") + ): + return f"Vertex AI context caching: model {deployment_model} does not support explicit context caching." + if not cached_messages_end_on_supported_turn(cached_messages): return ( "Vertex AI context caching: cached message block ends on a model turn once " @@ -537,6 +545,17 @@ class ContextCachingEndpoints(VertexBase): ) response.raise_for_status() except httpx.HTTPStatusError as err: + if "does not support explicit context caching" in err.response.text or ( + err.response.status_code == 400 + and "explicit" in err.response.text.lower() + and "caching" in err.response.text.lower() + ): + verbose_logger.warning( + "Model %s does not support explicit context caching (%s). Falling back to non-cached request.", + model, + err.response.text, + ) + return messages, optional_params, None error_code: Final = err.response.status_code raise VertexAIError(status_code=error_code, message=err.response.text) except httpx.TimeoutException: @@ -689,6 +708,17 @@ class ContextCachingEndpoints(VertexBase): ) response.raise_for_status() except httpx.HTTPStatusError as err: + if "does not support explicit context caching" in err.response.text or ( + err.response.status_code == 400 + and "explicit" in err.response.text.lower() + and "caching" in err.response.text.lower() + ): + verbose_logger.warning( + "Model %s does not support explicit context caching (%s). Falling back to non-cached request.", + model, + err.response.text, + ) + return None error_code: Final = err.response.status_code raise VertexAIError(status_code=error_code, message=err.response.text) except httpx.TimeoutException: diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 3244e0ae15b..9ea434734a1 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1398,6 +1398,11 @@ async def async_transform_request_body( context_caching_endpoints: Final = ContextCachingEndpoints() + if context_caching: + model_info = litellm_params.get("model_info") or {} + if model_info.get("supports_prompt_caching") is False: + context_caching = False + ( messages, optional_params, diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py new file mode 100644 index 00000000000..8451373409d --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py @@ -0,0 +1,162 @@ +import json +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.llms.anthropic.messages.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( + ContextCachingEndpoints, +) + + +def test_target_consumes_cache_control_skips_when_model_disables_prompt_caching(monkeypatch): + """LiteLLMAnthropicMessagesAdapter should not consume cache_control for models explicitly disabling prompt caching.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("claude-3-5-sonnet-20241022") is True + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-2.5-flash") is True + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-4-argon") is False + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("vertex_ai/gemini-4-argon") is False + + +def test_check_and_create_cache_skips_when_model_disables_prompt_caching(monkeypatch): + """ContextCachingEndpoints should skip explicit cache creation when model has supports_prompt_caching: false.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + endpoints = ContextCachingEndpoints() + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "hello world test token count " * 1000, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + optional_params = {"tools": [{"type": "function", "function": {"name": "test"}}]} + + res_messages, res_params, cache_id = endpoints.check_and_create_cache( + messages=messages, + optional_params=optional_params, + api_key="test-key", + api_base=None, + model="gemini/gemini-4-argon", + client=MagicMock(), + timeout=10, + logging_obj=MagicMock(), + custom_llm_provider="gemini", + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + assert cache_id is None + # Tools should remain in params and not be popped + assert "tools" in res_params + + +@pytest.mark.asyncio +async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): + """When Vertex AI returns 400 'does not support explicit context caching', gracefully fallback to non-cached request.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + endpoints = ContextCachingEndpoints() + endpoints.async_check_cache = AsyncMock(return_value=None) + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "hello world test token count " * 1000, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + optional_params = {"tools": [{"type": "function", "function": {"name": "test"}}]} + + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = json.dumps({ + "error": { + "code": 400, + "message": "Model gemini-4-argon does not support explicit context caching.", + "status": "INVALID_ARGUMENT", + } + }) + mock_client.post = AsyncMock( + side_effect=httpx.HTTPStatusError("Bad Request", request=MagicMock(), response=mock_response) + ) + + res_messages, res_params, cache_id = await endpoints.async_check_and_create_cache( + messages=messages, + optional_params=optional_params, + api_key="test-key", + api_base=None, + model="gemini/gemini-2.5-flash", + client=mock_client, + timeout=10, + logging_obj=MagicMock(), + custom_llm_provider="gemini", + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + assert cache_id is None + assert mock_client.post.called + + +def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): + """Sync check_and_create_cache fallback on 400 explicit caching unsupported.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + endpoints = ContextCachingEndpoints() + endpoints.check_cache = MagicMock(return_value=None) + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "hello world test token count " * 1000, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + optional_params = {"tools": [{"type": "function", "function": {"name": "test"}}]} + + mock_client = MagicMock(spec=HTTPHandler) + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = json.dumps({ + "error": { + "code": 400, + "message": "Model gemini-4-argon does not support explicit context caching.", + "status": "INVALID_ARGUMENT", + } + }) + mock_client.post.side_effect = httpx.HTTPStatusError( + "Bad Request", request=MagicMock(), response=mock_response + ) + + res_messages, res_params, cache_id = endpoints.check_and_create_cache( + messages=messages, + optional_params=optional_params, + api_key="test-key", + api_base=None, + model="gemini/gemini-2.5-flash", + client=mock_client, + timeout=10, + logging_obj=MagicMock(), + custom_llm_provider="gemini", + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + assert cache_id is None + assert mock_client.post.called From 2095cf41bb37ed097342830baeb9c485dcf83cb4 Mon Sep 17 00:00:00 2001 From: scriptsmay Date: Tue, 3 Nov 2026 20:09:18 +0800 Subject: [PATCH 3/3] fix(vertex_ai): preserve messages and tools on fallback, support bare models and sync transform --- .../messages/adapters/transformation.py | 7 +++ .../vertex_ai_context_caching.py | 10 +++- .../llms/vertex_ai/gemini/transformation.py | 5 ++ ..._vertex_ai_explicit_caching_unsupported.py | 58 ++++++++++++++++--- 4 files changed, 69 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b027f636c9..95ccd85fd4e 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -740,6 +740,13 @@ class LiteLLMAnthropicMessagesAdapter: if "gemini" in model.lower(): from litellm.utils import is_explicitly_disabled_factory + if "/" not in model: + if ( + is_explicitly_disabled_factory(f"gemini/{model}", None, "supports_prompt_caching") + or is_explicitly_disabled_factory(f"vertex_ai/{model}", None, "supports_prompt_caching") + ): + return False + if is_explicitly_disabled_factory(model, None, "supports_prompt_caching"): return False return True diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index b23f29340de..cea7f78b10a 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -507,9 +507,9 @@ class ContextCachingEndpoints(VertexBase): except _CachedContentsNotServedError: verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) return messages, optional_params, None - for tool_param in _TOOL_PARAMS: - optional_params.pop(tool_param, None) if google_cache_name: + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST @@ -571,6 +571,8 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached.get("name"), raw_response_cached.get("expireTime"), ) + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return ( non_cached_messages, optional_params, @@ -738,6 +740,10 @@ class ContextCachingEndpoints(VertexBase): except _CachedContentsNotServedError: verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) return messages, optional_params, None + + if cache_name is None: + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 9ea434734a1..a312ce904a8 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1262,6 +1262,11 @@ def sync_transform_request_body( context_caching_endpoints: Final = ContextCachingEndpoints() + if context_caching: + model_info = litellm_params.get("model_info") or {} + if model_info.get("supports_prompt_caching") is False: + context_caching = False + ( messages, optional_params, diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py index 8451373409d..5c0a6e58db4 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py @@ -11,20 +11,23 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) +from litellm.llms.vertex_ai.gemini.transformation import ( + sync_transform_request_body, +) -def test_target_consumes_cache_control_skips_when_model_disables_prompt_caching(monkeypatch): +def test_target_consumes_cache_control_skips_when_model_disables_prompt_caching(local_model_cost_map): """LiteLLMAnthropicMessagesAdapter should not consume cache_control for models explicitly disabling prompt caching.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("claude-3-5-sonnet-20241022") is True assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-2.5-flash") is True assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-4-argon") is False assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("vertex_ai/gemini-4-argon") is False + # Bare model name without provider prefix should also be recognized + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini-4-argon") is False -def test_check_and_create_cache_skips_when_model_disables_prompt_caching(monkeypatch): +def test_check_and_create_cache_skips_when_model_disables_prompt_caching(local_model_cost_map): """ContextCachingEndpoints should skip explicit cache creation when model has supports_prompt_caching: false.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() messages = [ { @@ -55,14 +58,14 @@ def test_check_and_create_cache_skips_when_model_disables_prompt_caching(monkeyp vertex_auth_header=None, ) assert cache_id is None - # Tools should remain in params and not be popped + # All messages and tools should remain intact + assert len(res_messages) == len(messages) assert "tools" in res_params @pytest.mark.asyncio -async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): +async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(local_model_cost_map): """When Vertex AI returns 400 'does not support explicit context caching', gracefully fallback to non-cached request.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() endpoints.async_check_cache = AsyncMock(return_value=None) messages = [ @@ -109,11 +112,13 @@ async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_uns ) assert cache_id is None assert mock_client.post.called + # Critical checks: messages must not be dropped, tools must not be stripped! + assert len(res_messages) == len(messages) + assert "tools" in res_params -def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): +def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(local_model_cost_map): """Sync check_and_create_cache fallback on 400 explicit caching unsupported.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() endpoints.check_cache = MagicMock(return_value=None) messages = [ @@ -160,3 +165,38 @@ def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupporte ) assert cache_id is None assert mock_client.post.called + # Critical checks: messages must not be dropped, tools must not be stripped! + assert len(res_messages) == len(messages) + assert "tools" in res_params + + +def test_sync_transform_request_body_skips_caching_on_model_info_override(monkeypatch): + """sync_transform_request_body should respect supports_prompt_caching: false in model_info.""" + mock_endpoints = MagicMock() + mock_endpoints.check_and_create_cache.return_value = ([], {}, None) + monkeypatch.setattr( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.ContextCachingEndpoints", + MagicMock(return_value=mock_endpoints), + ) + + messages = [{"role": "user", "content": "hello"}] + sync_transform_request_body( + messages=messages, + model="gemini-2.5-flash", + custom_llm_provider="gemini", + litellm_params={"model_info": {"supports_prompt_caching": False}}, + optional_params={}, + gemini_api_key="test-key", + api_base=None, + client=None, + timeout=None, + extra_headers=None, + logging_obj=MagicMock(), + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + # context_caching=False must be passed to check_and_create_cache + assert mock_endpoints.check_and_create_cache.called + _, kwargs = mock_endpoints.check_and_create_cache.call_args + assert kwargs.get("context_caching") is False