diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b027f636c9..95ccd85fd4e 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -740,6 +740,13 @@ class LiteLLMAnthropicMessagesAdapter: if "gemini" in model.lower(): from litellm.utils import is_explicitly_disabled_factory + if "/" not in model: + if ( + is_explicitly_disabled_factory(f"gemini/{model}", None, "supports_prompt_caching") + or is_explicitly_disabled_factory(f"vertex_ai/{model}", None, "supports_prompt_caching") + ): + return False + if is_explicitly_disabled_factory(model, None, "supports_prompt_caching"): return False return True diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index b23f29340de..cea7f78b10a 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -507,9 +507,9 @@ class ContextCachingEndpoints(VertexBase): except _CachedContentsNotServedError: verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) return messages, optional_params, None - for tool_param in _TOOL_PARAMS: - optional_params.pop(tool_param, None) if google_cache_name: + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return non_cached_messages, optional_params, google_cache_name ## TRANSFORM REQUEST @@ -571,6 +571,8 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached.get("name"), raw_response_cached.get("expireTime"), ) + for tool_param in _TOOL_PARAMS: + optional_params.pop(tool_param, None) return ( non_cached_messages, optional_params, @@ -738,6 +740,10 @@ class ContextCachingEndpoints(VertexBase): except _CachedContentsNotServedError: verbose_logger.debug("Gemini context caching: %s answers 404. Skipping context caching.", url) return messages, optional_params, None + + if cache_name is None: + return messages, optional_params, None + for tool_param in _TOOL_PARAMS: optional_params.pop(tool_param, None) return non_cached_messages, optional_params, cache_name diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 9ea434734a1..a312ce904a8 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1262,6 +1262,11 @@ def sync_transform_request_body( context_caching_endpoints: Final = ContextCachingEndpoints() + if context_caching: + model_info = litellm_params.get("model_info") or {} + if model_info.get("supports_prompt_caching") is False: + context_caching = False + ( messages, optional_params, diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py index 8451373409d..5c0a6e58db4 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_explicit_caching_unsupported.py @@ -11,20 +11,23 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) +from litellm.llms.vertex_ai.gemini.transformation import ( + sync_transform_request_body, +) -def test_target_consumes_cache_control_skips_when_model_disables_prompt_caching(monkeypatch): +def test_target_consumes_cache_control_skips_when_model_disables_prompt_caching(local_model_cost_map): """LiteLLMAnthropicMessagesAdapter should not consume cache_control for models explicitly disabling prompt caching.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("claude-3-5-sonnet-20241022") is True assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-2.5-flash") is True assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini/gemini-4-argon") is False assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("vertex_ai/gemini-4-argon") is False + # Bare model name without provider prefix should also be recognized + assert LiteLLMAnthropicMessagesAdapter.target_consumes_cache_control("gemini-4-argon") is False -def test_check_and_create_cache_skips_when_model_disables_prompt_caching(monkeypatch): +def test_check_and_create_cache_skips_when_model_disables_prompt_caching(local_model_cost_map): """ContextCachingEndpoints should skip explicit cache creation when model has supports_prompt_caching: false.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() messages = [ { @@ -55,14 +58,14 @@ def test_check_and_create_cache_skips_when_model_disables_prompt_caching(monkeyp vertex_auth_header=None, ) assert cache_id is None - # Tools should remain in params and not be popped + # All messages and tools should remain intact + assert len(res_messages) == len(messages) assert "tools" in res_params @pytest.mark.asyncio -async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): +async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(local_model_cost_map): """When Vertex AI returns 400 'does not support explicit context caching', gracefully fallback to non-cached request.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() endpoints.async_check_cache = AsyncMock(return_value=None) messages = [ @@ -109,11 +112,13 @@ async def test_async_check_and_create_cache_fallback_on_400_explicit_caching_uns ) assert cache_id is None assert mock_client.post.called + # Critical checks: messages must not be dropped, tools must not be stripped! + assert len(res_messages) == len(messages) + assert "tools" in res_params -def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(monkeypatch): +def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupported(local_model_cost_map): """Sync check_and_create_cache fallback on 400 explicit caching unsupported.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") endpoints = ContextCachingEndpoints() endpoints.check_cache = MagicMock(return_value=None) messages = [ @@ -160,3 +165,38 @@ def test_sync_check_and_create_cache_fallback_on_400_explicit_caching_unsupporte ) assert cache_id is None assert mock_client.post.called + # Critical checks: messages must not be dropped, tools must not be stripped! + assert len(res_messages) == len(messages) + assert "tools" in res_params + + +def test_sync_transform_request_body_skips_caching_on_model_info_override(monkeypatch): + """sync_transform_request_body should respect supports_prompt_caching: false in model_info.""" + mock_endpoints = MagicMock() + mock_endpoints.check_and_create_cache.return_value = ([], {}, None) + monkeypatch.setattr( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.ContextCachingEndpoints", + MagicMock(return_value=mock_endpoints), + ) + + messages = [{"role": "user", "content": "hello"}] + sync_transform_request_body( + messages=messages, + model="gemini-2.5-flash", + custom_llm_provider="gemini", + litellm_params={"model_info": {"supports_prompt_caching": False}}, + optional_params={}, + gemini_api_key="test-key", + api_base=None, + client=None, + timeout=None, + extra_headers=None, + logging_obj=MagicMock(), + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + # context_caching=False must be passed to check_and_create_cache + assert mock_endpoints.check_and_create_cache.called + _, kwargs = mock_endpoints.check_and_create_cache.call_args + assert kwargs.get("context_caching") is False