diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index f2491a7a490..f5ed0f73160 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -448,7 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite and later accept the fields but ignore them, and a later generation will 400 on them. A chat model the map says nothing about is judged by the version in its name, so a model - newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding. + newer than the map keeps the same contract. A batch deployment's rows are chat requests and + the proxy stamps its ``mode: batch`` onto the backend row, so that row is judged the same way; + a row for any other mode without the flag keeps forwarding. ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and @@ -461,7 +463,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): for row in matched: if isinstance(flag := row.get("supports_sampling_params"), bool): return flag - if any(row.get("mode") not in (None, "chat") for row in matched): + if any(row.get("mode") not in (None, "chat", "batch") for row in matched): return True return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index cf4a7c4cbc8..ec46a24cc40 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -171,9 +171,7 @@ class TestBatchObjectNaming: def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" - object_name = config._get_gcs_object_name_from_batch_jsonl( - [{"body": {"model": "gemini/7768560373388541952"}}] - ) + object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}]) assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/") assert "publishers" not in object_name @@ -865,7 +863,6 @@ class TestVertexBatchOutputTransformation: assert config._try_transform_vertex_batch_output_to_openai(binary) == binary - class TestTryTransformDoesNotMutateCallerLoggingObj: """Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate the caller's logging_obj (model, start_time, optional_params).""" @@ -2045,3 +2042,27 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py monkeypatch.setattr(litellm, "drop_params", False) (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS} + + +def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None: + monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"}) + + +def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini + model whose name is past Google's cut must still be judged by its version and not pass as a non-chat row""" + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"): + _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + + +def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + assert _generation_config(row) == {"max_output_tokens": 5} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8b5787006be..ad61026ac59 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3941,16 +3941,24 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( [ ({"mode": "chat"}, False), ({}, False), + ({"mode": "batch"}, False), ({"mode": "audio_speech"}, True), ({"mode": "chat", "supports_sampling_params": True}, True), ], - ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"], + ids=[ + "chat_row_without_flag", + "row_without_mode", + "batch_deployment_row_without_flag", + "non_chat_row_without_flag", + "explicit_true", + ], ) def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule( row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch ) -> None: - """An explicit flag always wins, a row silent on it is judged by the version, and a row for another - mode keeps forwarding because Google's notice covers the text models""" + """An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment + is judged by the version, and a row for another mode keeps forwarding because Google's notice covers + the text models""" from litellm.utils import get_optional_params monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)