From 7fc54ad460938ee81eae021b510db4dddb0bb2ee Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 20 Apr 2026 10:23:40 +0530 Subject: [PATCH 1/3] fix(vertex_ai): omit system_instruction/tools/toolConfig when cachedContent set Vertex generateContent returns INVALID_ARGUMENT if cachedContent is sent with system_instruction, tools, or toolConfig; those belong on CachedContent. Fixes #26014 Made-with: Cursor --- .../llms/vertex_ai/gemini/transformation.py | 9 ++-- .../test_vertex_ai_gemini_transformation.py | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index d0f0e5c1e24..ef35fc09610 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -748,13 +748,14 @@ def _transform_request_body( # noqa: PLR0915 ] data = RequestBody(contents=content) - if system_instructions is not None: + # Vertex rejects system_instruction/tools/toolConfig alongside cachedContent. + if system_instructions is not None and cached_content is None: data["system_instruction"] = system_instructions - if tools is not None: + if tools is not None and cached_content is None: data["tools"] = tools - if tool_choice is not None: + if tool_choice is not None and cached_content is None: data["toolConfig"] = tool_choice - if include_server_side_tool_invocations: + if include_server_side_tool_invocations and cached_content is None: if "toolConfig" not in data: data["toolConfig"] = {} data["toolConfig"]["includeServerSideToolInvocations"] = True diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 6937b4c3ba1..6952be7db3c 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -86,6 +86,53 @@ def test_check_if_part_exists_in_parts_camel_case_snake_case(): assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) +def test_cached_content_omits_system_instruction_tools_toolconfig(): + """Regression: #26014 / #17304 — cachedContent must not ship with tools/system/toolConfig.""" + cache_name = "projects/p/locations/us-central1/cachedContents/abc123" + messages = [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "hi"}, + ] + optional_params = { + "tools": [ + { + "functionDeclarations": [ + {"name": "get_weather", "description": "Get weather"}, + ] + } + ], + "tool_choice": {"functionCallingConfig": {"mode": "AUTO"}}, + } + + result = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=cache_name, + ) + + assert result.get("cachedContent") == cache_name + assert "system_instruction" not in result + assert "tools" not in result + assert "toolConfig" not in result + assert "contents" in result + + # Without cache, conflicting fields are included as before + result_no_cache = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=None, + ) + assert "system_instruction" in result_no_cache + assert "tools" in result_no_cache + assert "toolConfig" in result_no_cache + + # Tests for issue #14556: Labels field provider-aware filtering def test_google_genai_excludes_labels(): """Test that Google GenAI/AI Studio endpoints exclude labels when custom_llm_provider='gemini'""" From 47614f967bed2966f52a27a7428c1f32c744a7a4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 22 Apr 2026 17:47:27 +0530 Subject: [PATCH 2/3] refactor code --- .../llms/vertex_ai/gemini/transformation.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index ef35fc09610..33301a8fd8d 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -749,16 +749,21 @@ def _transform_request_body( # noqa: PLR0915 data = RequestBody(contents=content) # Vertex rejects system_instruction/tools/toolConfig alongside cachedContent. - if system_instructions is not None and cached_content is None: - data["system_instruction"] = system_instructions - if tools is not None and cached_content is None: - data["tools"] = tools - if tool_choice is not None and cached_content is None: - data["toolConfig"] = tool_choice - if include_server_side_tool_invocations and cached_content is None: - if "toolConfig" not in data: - data["toolConfig"] = {} - data["toolConfig"]["includeServerSideToolInvocations"] = True + # Treat dropping these fields as a request mutation guarded by modify_params. + can_send_cache_incompatible_fields = ( + cached_content is None or litellm.modify_params is False + ) + if can_send_cache_incompatible_fields: + if system_instructions is not None: + data["system_instruction"] = system_instructions + if tools is not None: + data["tools"] = tools + if tool_choice is not None: + data["toolConfig"] = tool_choice + if include_server_side_tool_invocations: + if "toolConfig" not in data: + data["toolConfig"] = {} + data["toolConfig"]["includeServerSideToolInvocations"] = True if safety_settings is not None: data["safetySettings"] = safety_settings if generation_config is not None and len(generation_config) > 0: From e03bb3437feb03d3f91a9cdbaa45b942e1351a6b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 24 Apr 2026 20:10:30 +0530 Subject: [PATCH 3/3] Fix test --- .../test_vertex_ai_gemini_transformation.py | 77 ++++++++++++------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 6952be7db3c..d09c1afcdb5 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -86,8 +86,10 @@ def test_check_if_part_exists_in_parts_camel_case_snake_case(): assert check_if_part_exists_in_parts(parts_mixed, part_mixed_casing) -def test_cached_content_omits_system_instruction_tools_toolconfig(): - """Regression: #26014 / #17304 — cachedContent must not ship with tools/system/toolConfig.""" +def test_cached_content_respects_modify_params_for_cache_incompatible_fields(): + """Regression: cachedContent drops system/tools/toolConfig only when modify_params=True.""" + import litellm + cache_name = "projects/p/locations/us-central1/cachedContents/abc123" messages = [ {"role": "system", "content": "You are helpful"}, @@ -104,33 +106,54 @@ def test_cached_content_omits_system_instruction_tools_toolconfig(): "tool_choice": {"functionCallingConfig": {"mode": "AUTO"}}, } - result = _transform_request_body( - messages=list(messages), - model="gemini-2.5-pro", - optional_params=dict(optional_params), - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=cache_name, - ) + original_modify_params = litellm.modify_params + try: + # With modify_params=False (default), keep fields even with cachedContent. + litellm.modify_params = False + result = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=cache_name, + ) + assert result.get("cachedContent") == cache_name + assert "system_instruction" in result + assert "tools" in result + assert "toolConfig" in result + assert "contents" in result - assert result.get("cachedContent") == cache_name - assert "system_instruction" not in result - assert "tools" not in result - assert "toolConfig" not in result - assert "contents" in result + # With modify_params=True, drop cache-incompatible fields. + litellm.modify_params = True + result_modify_true = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=cache_name, + ) + assert result_modify_true.get("cachedContent") == cache_name + assert "system_instruction" not in result_modify_true + assert "tools" not in result_modify_true + assert "toolConfig" not in result_modify_true + assert "contents" in result_modify_true - # Without cache, conflicting fields are included as before - result_no_cache = _transform_request_body( - messages=list(messages), - model="gemini-2.5-pro", - optional_params=dict(optional_params), - custom_llm_provider="vertex_ai", - litellm_params={}, - cached_content=None, - ) - assert "system_instruction" in result_no_cache - assert "tools" in result_no_cache - assert "toolConfig" in result_no_cache + # Without cache, fields are always included. + result_no_cache = _transform_request_body( + messages=list(messages), + model="gemini-2.5-pro", + optional_params=dict(optional_params), + custom_llm_provider="vertex_ai", + litellm_params={}, + cached_content=None, + ) + assert "system_instruction" in result_no_cache + assert "tools" in result_no_cache + assert "toolConfig" in result_no_cache + finally: + litellm.modify_params = original_modify_params # Tests for issue #14556: Labels field provider-aware filtering