From 7b8ab61415a23df02b697a8f6fa4396f02416eb0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 16:52:52 -0700 Subject: [PATCH 1/7] feat(gemini)!: judge Gemini models the map has no sampling flag for by the version in their name Google ignores temperature, top_p and top_k on Gemini 3.6 Flash, 3.5 Flash-Lite and every later model. The model map carries supports_sampling_params: false for the rows we know about, but a chat row without the flag, or a model newer than the map, kept forwarding the knobs silently. When no row answers, read the version out of the model name and apply the same drop-or-400 contract. A row whose mode is not chat keeps forwarding, since the notice covers the text models. gemini-4-argon has a chat row without the flag, so it now drops the knobs under drop_params and returns the 400 naming drop_params without it, like the flagged rows. --- .../vertex_and_google_ai_studio_gemini.py | 34 +++++++--- ...test_vertex_and_google_ai_studio_gemini.py | 66 ++++++++++++++++++- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 241b2d2a3a0..f2491a7a490 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -429,23 +429,41 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False return True + @staticmethod + def _google_ignores_sampling_params_by_version(model_name: str) -> bool: + """Google's announced cut: Gemini 3.6 Flash, 3.5 Flash-Lite and every later model ignore the + sampling knobs, so a name with such a version is judged by the version when the map has no answer.""" + match: Final = re.match(r"^gemini-(\d+)(?:\.(\d+))?(?:-|$)", model_name.lower()) + if match is None: + return False + version: Final = (int(match.group(1)), int(match.group(2) or 0)) + if version >= (3, 6): + return True + return version == (3, 5) and "flash-lite" in model_name.lower() + @staticmethod def supports_sampling_params(model: str) -> bool: """Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``. Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite and later accept the fields but ignore them, and a later generation will 400 on them. - A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p`` - leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param - check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; - ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the - request body is built.""" + A chat model the map says nothing about is judged by the version in its name, so a model + newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding. + ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so + the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still + forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and + is gated while the request body is built.""" rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id bare: Final = model.split("/")[-1] - for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"): - if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool): + matched: Final = tuple( + row for key in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}") if (row := rows.get(key)) is not None + ) + for row in matched: + if isinstance(flag := row.get("supports_sampling_params"), bool): return flag - return True + if any(row.get("mode") not in (None, "chat") for row in matched): + return True + return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare) @staticmethod def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None: diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bb624add673..8b5787006be 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3750,12 +3750,13 @@ def _gemini_request_body( custom_llm_provider: Literal["vertex_ai", "gemini"], optional_params: dict[str, object], litellm_params: dict[str, object], + model: str = "gemini-zeta-9", ) -> RequestBody: from litellm.llms.vertex_ai.gemini.transformation import transform_request_body return transform_request_body( messages=[{"role": "user", "content": "hi"}], - model="gemini-zeta-9", + model=model, optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, @@ -3902,6 +3903,65 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un assert optional_params["max_output_tokens"] == 10 +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +@pytest.mark.parametrize( + ("model", "forwarded"), + [ + ("gemini-3.6-flash-zeta", False), + ("gemini-3.5-flash-lite-zeta", False), + ("gemini-3.7-pro-zeta", False), + ("gemini-4-zeta", False), + ("gemini-4.1-flash-zeta", False), + ("gemini-10-zeta", False), + ("gemini-3.5-flash-zeta", True), + ("gemini-3.5-pro-zeta", True), + ("gemini-3.1-pro-zeta", True), + ("gemini-2.5-flash-zeta", True), + ("gemini-zeta-9", True), + ], +) +def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( + custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool +) -> None: + """Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer + than the map gets the same drop-or-400 contract instead of silently forwarding (#47015 follow-up)""" + from litellm.utils import get_optional_params + + optional_params: Final = get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS + ) + body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model) + + knobs: Final = _SAMPLING_PARAMS if forwarded else {} + assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model + + +@pytest.mark.parametrize( + ("row", "forwarded"), + [ + ({"mode": "chat"}, False), + ({}, False), + ({"mode": "audio_speech"}, True), + ({"mode": "chat", "supports_sampling_params": True}, True), + ], + ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"], +) +def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule( + row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """An explicit flag always wins, a row silent on it is judged by the version, and a row for another + mode keeps forwarding because Google's notice covers the text models""" + from litellm.utils import get_optional_params + + monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row) + + optional_params: Final = get_optional_params( + model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True + ) + + assert ("temperature" in optional_params) is forwarded + + def test_media_resolution_from_detail_parameter(): """Test that OpenAI's detail parameter is correctly mapped to media_resolution""" from litellm.llms.vertex_ai.gemini.transformation import ( @@ -6928,7 +6988,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( caplog: pytest.LogCaptureFixture, ) -> None: - unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100)) mapped: Final = _mapped_gemini_3_models()[0] config: Final = VertexGeminiConfig() @@ -6971,7 +7031,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20)) for name in priced: monkeypatch.setitem( litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed From c01d4acffa111ac3c738c4be41a1e9b6fbe3043e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 17:14:30 -0700 Subject: [PATCH 2/7] fix(gemini): judge a batch deployment's backend row by the version rule too The proxy registers a batch deployment's model_info onto the backend model's map row, so vertex_ai/gemini-4-argon carried mode: batch on a batch proxy and the version rule treated it as a non-chat row and kept forwarding the knobs. A live upload on the PR tip stored temperature, top_p and top_k in the bucket on both legs. Batch rows are chat requests, so a batch row is judged by the version like a chat row, and the batch transformation now has the regression --- .../vertex_and_google_ai_studio_gemini.py | 6 ++-- .../test_vertex_ai_files_transformation.py | 29 ++++++++++++++++--- ...test_vertex_and_google_ai_studio_gemini.py | 14 +++++++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index f2491a7a490..f5ed0f73160 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -448,7 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite and later accept the fields but ignore them, and a later generation will 400 on them. A chat model the map says nothing about is judged by the version in its name, so a model - newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding. + newer than the map keeps the same contract. A batch deployment's rows are chat requests and + the proxy stamps its ``mode: batch`` onto the backend row, so that row is judged the same way; + a row for any other mode without the flag keeps forwarding. ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and @@ -461,7 +463,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): for row in matched: if isinstance(flag := row.get("supports_sampling_params"), bool): return flag - if any(row.get("mode") not in (None, "chat") for row in matched): + if any(row.get("mode") not in (None, "chat", "batch") for row in matched): return True return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index cf4a7c4cbc8..ec46a24cc40 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -171,9 +171,7 @@ class TestBatchObjectNaming: def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" - object_name = config._get_gcs_object_name_from_batch_jsonl( - [{"body": {"model": "gemini/7768560373388541952"}}] - ) + object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}]) assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/") assert "publishers" not in object_name @@ -865,7 +863,6 @@ class TestVertexBatchOutputTransformation: assert config._try_transform_vertex_batch_output_to_openai(binary) == binary - class TestTryTransformDoesNotMutateCallerLoggingObj: """Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate the caller's logging_obj (model, start_time, optional_params).""" @@ -2045,3 +2042,27 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py monkeypatch.setattr(litellm, "drop_params", False) (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS} + + +def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None: + monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"}) + + +def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini + model whose name is past Google's cut must still be judged by its version and not pass as a non-chat row""" + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"): + _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + + +def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + assert _generation_config(row) == {"max_output_tokens": 5} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8b5787006be..ad61026ac59 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3941,16 +3941,24 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( [ ({"mode": "chat"}, False), ({}, False), + ({"mode": "batch"}, False), ({"mode": "audio_speech"}, True), ({"mode": "chat", "supports_sampling_params": True}, True), ], - ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"], + ids=[ + "chat_row_without_flag", + "row_without_mode", + "batch_deployment_row_without_flag", + "non_chat_row_without_flag", + "explicit_true", + ], ) def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule( row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch ) -> None: - """An explicit flag always wins, a row silent on it is judged by the version, and a row for another - mode keeps forwarding because Google's notice covers the text models""" + """An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment + is judged by the version, and a row for another mode keeps forwarding because Google's notice covers + the text models""" from litellm.utils import get_optional_params monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row) From 651b04bf40e226fae28445a054e855f9eb2b783f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 17:18:58 -0700 Subject: [PATCH 3/7] fix(gemini): stop the sampling 400 from claiming a map flag the version rule never set A model judged by its version has no supports_sampling_params flag in the map, so the batch and top_k rejection said something a reader could not find. The message now says Google ignores the knobs and names both escape hatches --- .../vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index f5ed0f73160..e7eb8334ffc 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -475,9 +475,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return raise UnsupportedParamsError( message=( - f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` " - "because Google ignores sampling params on this model. " - "To drop unsupported params, set `litellm.drop_params = True`." + f"{model} does not support {param}: Google ignores sampling params on this model, so LiteLLM " + "treats it as `supports_sampling_params: false`. To drop unsupported params, set " + "`litellm.drop_params = True`, or set `supports_sampling_params: true` in the model's " + "model_info to forward them anyway." ), status_code=400, ) From 75c9dd1c4b6d0b825695066130fadaf1c4276c58 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 24 Oct 2026 15:17:56 -0700 Subject: [PATCH 4/7] refactor(gemini): ship the sampling cut as a fallback generalization rule The hand-rolled version regex and raw model map lookup in the Vertex config move into a gemini-sampling-params-ignored rule in the model map. The rule fills supports_sampling_params false onto Google rows that lack the flag and resolves unmapped Gemini 3.6+, 3.5 flash-lite and 4+ ids, while an explicit flag on a row still wins The Gemini and Vertex configs now read the flag through the shared tri-state reader for their own route, so a batch deployment whose stamped row says mode batch is judged by the same rule as a chat row. Fill-missing backfill now also covers rows whose mode is batch and rows without a litellm_provider, and get_model_info returns supports_sampling_params --- litellm/llms/gemini/chat/transformation.py | 8 +- .../llms/vertex_ai/files/transformation.py | 2 +- .../llms/vertex_ai/gemini/transformation.py | 4 +- .../vertex_and_google_ai_studio_gemini.py | 52 ++++--------- ...odel_prices_and_context_window_backup.json | 9 +++ litellm/utils.py | 7 +- model_prices_and_context_window.json | 9 +++ .../test_fallback_generalizations.py | 78 ++++++++++++++++++- .../test_vertex_ai_files_transformation.py | 13 ++-- ...test_vertex_and_google_ai_studio_gemini.py | 27 +++++-- 10 files changed, 155 insertions(+), 54 deletions(-) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 936919af83d..9d757745a6d 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -81,8 +81,14 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): def is_model_gemini_audio_model(self, model: str) -> bool: return "tts" in model + @property + def custom_llm_provider(self) -> str: + return "gemini" + def get_supported_openai_params(self, model: str) -> list[str]: - sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else () + sampling_params: Final = ( + ("temperature", "top_p") if self.supports_sampling_params(model, self.custom_llm_provider) else () + ) supported_params: Final = [ *sampling_params, "max_tokens", diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 176569f5d8d..881c6d56b69 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -1113,7 +1113,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): sampling_params: Final = tuple( p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None ) - if sampling_params and not VertexGeminiConfig.supports_sampling_params(model): + if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"): VertexGeminiConfig.reject_unsupported_sampling_param( model=model, param=", ".join(sampling_params), drop_params=False ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index e5ce395031a..ea50944f40f 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1192,7 +1192,9 @@ def transform_request_body( from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig - drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model) + drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params( + model, custom_llm_provider + ) if drops_top_k: request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature VertexGeminiConfig.reject_unsupported_sampling_param( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index e7eb8334ffc..191ec24ce8e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -429,43 +429,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False return True - @staticmethod - def _google_ignores_sampling_params_by_version(model_name: str) -> bool: - """Google's announced cut: Gemini 3.6 Flash, 3.5 Flash-Lite and every later model ignore the - sampling knobs, so a name with such a version is judged by the version when the map has no answer.""" - match: Final = re.match(r"^gemini-(\d+)(?:\.(\d+))?(?:-|$)", model_name.lower()) - if match is None: - return False - version: Final = (int(match.group(1)), int(match.group(2) or 0)) - if version >= (3, 6): - return True - return version == (3, 5) and "flash-lite" in model_name.lower() + @property + def custom_llm_provider(self) -> str: + return "vertex_ai" @staticmethod - def supports_sampling_params(model: str) -> bool: + def supports_sampling_params(model: str, custom_llm_provider: str | None) -> bool: """Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``. - Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite - and later accept the fields but ignore them, and a later generation will 400 on them. - A chat model the map says nothing about is judged by the version in its name, so a model - newer than the map keeps the same contract. A batch deployment's rows are chat requests and - the proxy stamps its ``mode: batch`` onto the backend row, so that row is judged the same way; - a row for any other mode without the flag keeps forwarding. - ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so - the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still - forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and - is gated while the request body is built.""" - rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id - bare: Final = model.split("/")[-1] - matched: Final = tuple( - row for key in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}") if (row := rows.get(key)) is not None - ) - for row in matched: - if isinstance(flag := row.get("supports_sampling_params"), bool): - return flag - if any(row.get("mode") not in (None, "chat", "batch") for row in matched): - return True - return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare) + Read from the model map through the provider-aware lookup: an explicit ``supports_sampling_params`` + on the row wins, a chat, Responses or batch row without one and an id the map has never seen are + answered by the ``gemini-sampling-params-ignored`` fallback generalization (Gemini 3.6 Flash, 3.5 + Flash-Lite and every later text model), and a row for any other mode keeps forwarding. + ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the + shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards + them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while + the request body is built.""" + return not is_explicitly_disabled_factory(model, custom_llm_provider, "supports_sampling_params") @staticmethod def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None: @@ -484,7 +464,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) def get_supported_openai_params(self, model: str) -> list[str]: - sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else () + sampling_params: Final = ( + ("temperature", "top_p", "top_k") if self.supports_sampling_params(model, self.custom_llm_provider) else () + ) supported_params: Final = [ *sampling_params, "max_tokens", @@ -1344,7 +1326,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) - supports_sampling_params: Final = self.supports_sampling_params(model) + supports_sampling_params: Final = self.supports_sampling_params(model, self.custom_llm_provider) for param, value in non_default_params.items(): if param in ("temperature", "top_p", "top_k") and not supports_sampling_params: continue diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 98fee42de3c..52bddfca3e9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63592,6 +63592,15 @@ "supports_prompt_caching": true, "supports_web_search": true } + }, + { + "name": "gemini-sampling-params-ignored", + "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)", + "fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"], + "description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini--) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.", + "model_info": { + "supports_sampling_params": false + } } ] }, diff --git a/litellm/utils.py b/litellm/utils.py index 897138a940c..cc83ed0d504 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -270,7 +270,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} -_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) +_BACKFILL_MODES: Final = frozenset({"chat", "responses", "batch"}) # +-----------------------------------------------+ # | | @@ -6640,7 +6640,9 @@ def get_model_info_helper( _model_info: dict[str, Any] | None = lookup[1] if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", "")) + fill_missing: Final = match_fill_missing_generalizations( + key, _model_info.get("litellm_provider") or model_cost_custom_llm_provider or "" + ) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, @@ -6891,6 +6893,7 @@ def get_model_info_helper( supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), + supports_sampling_params=_model_info.get("supports_sampling_params", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 98fee42de3c..52bddfca3e9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63592,6 +63592,15 @@ "supports_prompt_caching": true, "supports_web_search": true } + }, + { + "name": "gemini-sampling-params-ignored", + "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)", + "fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"], + "description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini--) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.", + "model_info": { + "supports_sampling_params": false + } } ] }, diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index 7807c438503..13babf08056 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -390,6 +390,17 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "litellm_provider": "openrouter", "mode": "chat", }, + "acme-batch": { + "input_cost_per_token": 9e-6, + "output_cost_per_token": 1e-5, + "litellm_provider": "openai", + "mode": "batch", + }, + "acme-unowned": { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 3e-6, + "mode": "chat", + }, }, ) restore_generalizations( @@ -420,6 +431,14 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp image = litellm.get_model_info("acme-image", custom_llm_provider="openai") assert image.get("supports_reasoning") is None + batch = litellm.get_model_info("acme-batch", custom_llm_provider="openai") + assert batch["supports_reasoning"] is True + assert batch["max_tokens"] == 5 + + unowned = litellm.get_model_info("acme-unowned", custom_llm_provider="openai") + assert unowned["supports_reasoning"] is True + assert litellm.get_model_info("acme-unowned", custom_llm_provider="openrouter").get("supports_reasoning") is None + restore_generalizations( [{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}] ) @@ -504,7 +523,7 @@ def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cos "databricks-gemini-3-1-flash-image", "openrouter/google/gemini-2.0-flash-001", ): - assert match_capability_generalizations(model) is None, model + assert "mode" not in (match_capability_generalizations(model) or {}), model def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map): @@ -983,3 +1002,60 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None + + +@pytest.mark.parametrize( + "model,provider,sampling", + [ + ("gemini-3.6-flash-zeta", "gemini", False), + ("gemini-3.5-flash-lite-zeta", "vertex_ai", False), + ("gemini-3.10-pro-zeta", "gemini", False), + ("gemini-4-zeta", "gemini", False), + ("gemini-4.1-flash-zeta", "vertex_ai", False), + ("gemini-10-zeta", "gemini", False), + ("vertex_ai/gemini-4-zeta", None, False), + ("gemini-3.5-flash-zeta", "gemini", None), + ("gemini-3.5-pro-zeta", "vertex_ai", None), + ("gemini-3.1-pro-zeta", "gemini", None), + ("gemini-2.5-flash-zeta", "gemini", None), + ], +) +def test_shipped_gemini_sampling_rule_version_boundaries(shipped_cost_map, model, provider, sampling): + """The gemini-sampling-params-ignored rule marks Gemini 3.6 and every later 3.x minor, every 4+ major + and 3.5 Flash-Lite as not honoring temperature, top_p and top_k, and leaves 3.5 Flash, 3.5 Pro, 3.1 and + 2.5 ids without an opinion""" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info.get("supports_sampling_params") is sampling, model + + +def test_shipped_gemini_sampling_rule_skips_non_text_ids(shipped_cost_map): + for model in ( + "gemini/gemini-4-flash-image", + "gemini/gemini-3.8-flash-tts", + "gemini/gemini-4-flash-live-preview", + "gemini/gemini-3.6-flash-native-audio", + "gemini/gemini-embedding-4", + "gemini/gemini-4-flash-transcribe", + "gemini/gemini-3.6-flash-live-translate-preview", + "gemini/gemini-4-computer-use-preview", + "gemini/gemini-zeta-9", + ): + assert (match_capability_generalizations(model) or {}).get("supports_sampling_params") is None, model + + +def test_shipped_gemini_sampling_rule_fills_mapped_google_rows_without_the_flag(shipped_cost_map): + """A mapped Gemini row past Google's cut reads as supports_sampling_params: false on AI Studio and Vertex + whether or not someone flagged it by hand, and a reseller copy of the same model is left alone""" + for key, model, provider in ( + ("gemini/gemini-4-argon", "gemini-4-argon", "gemini"), + ("vertex_ai/gemini-4-argon", "gemini-4-argon", "vertex_ai"), + ("gemini/gemini-3.6-flash", "gemini-3.6-flash", "gemini"), + ): + assert key in litellm.model_cost + assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_sampling_params"] is False, key + + assert match_fill_missing_generalizations("gemini-4-argon", "vertex_ai-language-models") == { + "supports_sampling_params": False + } + assert match_fill_missing_generalizations("gemini-4-argon", "github_copilot") is None diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index ec46a24cc40..64af8153169 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -6,14 +6,15 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse from collections.abc import Mapping -from typing import Final from types import MappingProxyType +from typing import Final +from unittest.mock import MagicMock from urllib.parse import parse_qs, urlparse import httpx import pytest +from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter -from unittest.mock import MagicMock import litellm from litellm.llms.vertex_ai.files.transformation import ( @@ -22,8 +23,7 @@ from litellm.llms.vertex_ai.files.transformation import ( _openai_batch_jsonl_entry_to_vertex_rows, _sanitize_gcp_label_value, ) -from litellm.types.llms.openai import OpenAIFileObject, HttpxBinaryResponseContent -from openai.types.file_deleted import FileDeleted +from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject @pytest.fixture @@ -2048,17 +2048,20 @@ def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"}) +@pytest.mark.usefixtures("local_model_cost_map") def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params( monkeypatch: pytest.MonkeyPatch, ) -> None: """The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini - model whose name is past Google's cut must still be judged by its version and not pass as a non-chat row""" + model whose name is past Google's cut must still be filled by the shipped sampling rule and not pass as a + non-chat row""" _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") monkeypatch.setattr(litellm, "drop_params", False) with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"): _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) +@pytest.mark.usefixtures("local_model_cost_map") def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index ad61026ac59..35818974780 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3852,13 +3852,21 @@ def test_gemini_keeps_forwarding_sampling_params_unless_the_model_map_flags_them assert body["generationConfig"] == _SAMPLING_PARAMS -@pytest.mark.parametrize("model_key", ["gemini-zeta-9", "vertex_ai/gemini-zeta-9", "gemini/gemini-zeta-9"]) -def test_gemini_sampling_flag_is_read_from_whichever_provider_row_carries_it( - model_key: str, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("config_cls", "model_key"), + [ + ("vertex", "gemini-zeta-9"), + ("vertex", "vertex_ai/gemini-zeta-9"), + ("studio", "gemini-zeta-9"), + ("studio", "gemini/gemini-zeta-9"), + ], +) +def test_gemini_sampling_flag_is_read_from_the_row_of_the_route_serving_the_call( + config_cls: Literal["vertex", "studio"], model_key: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setitem(litellm.model_cost, model_key, {"supports_sampling_params": False}) - result: Final = VertexGeminiConfig().map_openai_params( + result: Final = _gemini_config(config_cls).map_openai_params( non_default_params={"temperature": 0.3}, optional_params={}, model="gemini-zeta-9", @@ -3903,6 +3911,7 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un assert optional_params["max_output_tokens"] == 10 +@pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) @pytest.mark.parametrize( ("model", "forwarded"), @@ -3920,11 +3929,12 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un ("gemini-zeta-9", True), ], ) -def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( +def test_gemini_model_unknown_to_the_map_is_judged_by_the_shipped_sampling_rule( custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool ) -> None: """Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer - than the map gets the same drop-or-400 contract instead of silently forwarding (#47015 follow-up)""" + than the map gets the same drop-or-400 contract from the gemini-sampling-params-ignored fallback + generalization instead of silently forwarding (#47015 follow-up)""" from litellm.utils import get_optional_params optional_params: Final = get_optional_params( @@ -3936,6 +3946,7 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model +@pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( ("row", "forwarded"), [ @@ -3953,11 +3964,11 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( "explicit_true", ], ) -def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule( +def test_map_row_for_a_gemini_4_model_decides_before_the_shipped_sampling_rule( row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch ) -> None: """An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment - is judged by the version, and a row for another mode keeps forwarding because Google's notice covers + is filled by the shipped rule, and a row for another mode keeps forwarding because Google's notice covers the text models""" from litellm.utils import get_optional_params From c2c76fc7e4eaad1d18721078b8a6c57643d6e210 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 24 Oct 2026 18:37:13 -0700 Subject: [PATCH 5/7] fix(vertex_ai): gate a batch row's sampling knobs by the model the job runs on A batch file row whose model kept a gemini/ prefix, copied from an AI Studio call, was looked up under that prefix on the Vertex route. The AI Studio row exists, so the provider check rejected it and the fallback rule never ran, the lookup raised, and the reader fell open: temperature, top_p and top_k were stored even under drop_params. The row runs on the deployment's Vertex model either way, which the stored GCS path already binds it to, so the row is now shaped and gated for that model, falling back to its own id only for a direct SDK upload that names no deployment, with the Google namespace stripped the way a live call strips it --- .../llms/vertex_ai/files/transformation.py | 40 +++++++++--- .../test_vertex_ai_files_transformation.py | 64 ++++++++++++++++++- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 881c6d56b69..250aec1a14c 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -679,9 +679,28 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/") + + +def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None: + configured_model: Final = litellm_params.get("model") + return configured_model if isinstance(configured_model, str) else None + + +def _batch_row_model(deployment_model: str | None, row_model: object) -> str: + """The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path + binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google + namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied + from an AI Studio call lands on its Vertex row instead of past every row.""" + chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else "" + namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") + return chosen.removeprefix(namespace) + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None = None, ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. @@ -701,10 +720,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))} vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), - model=chat_request_body.get("model", ""), - optional_params=map_openai_to_vertex_params(chat_request_body), + messages=map_developer_role_to_system_role(row_body.get("messages", [])), + model=row_body["model"], + optional_params=map_openai_to_vertex_params(row_body), custom_llm_provider="vertex_ai", litellm_params={}, cached_content=None, @@ -854,15 +874,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._deployment_model = deployment_model def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, self._map_openai_to_vertex_params, self._deployment_model + ) except Exception as e: raise VertexAIError( status_code=400, @@ -1047,7 +1071,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], data: CreateFileRequest, ) -> str: """ @@ -1071,11 +1095,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_configured_deployment_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1131,7 +1154,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1162,6 +1185,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, self._map_openai_to_vertex_params, + _configured_deployment_model(litellm_params), ), content_type="application/json", ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 64af8153169..b4482e4a413 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from types import MappingProxyType from typing import Final from unittest.mock import MagicMock @@ -17,6 +17,7 @@ from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter import litellm +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -2069,3 +2070,64 @@ def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_ monkeypatch.setattr(litellm, "drop_params", True) (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5} + + +def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model_name}", + {"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False}, + ) + + +def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]: + """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's + `?model=` branch runs it, with the deployment's model in litellm_params""" + jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + request: Final = VertexAIFilesConfig().transform_create_file_request( + model="", + create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, + optional_params={}, + litellm_params={"model": deployment_model} if deployment_model else {}, + ) + assert isinstance(request, dict) + body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) + return [json.loads(line) for line in body.decode().splitlines()] + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag + decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were + stored even though the model ignores them""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"): + _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + +def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that + still takes the knobs is gated for the model that actually executes it""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setitem( + litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"} + ) + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"): + _upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9") From 62ff5d2bd93d43d819104e1bb9179da62c26c2e1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 17:22:46 -0700 Subject: [PATCH 6/7] fix(vertex_ai): honor a batch deployment's drop_params on flagged sampling knobs A batch deployment with `litellm_params: drop_params: true` still rejected rows that carried temperature, top_p or top_k for a Gemini model flagged `supports_sampling_params: false`, because the batch mapper passed `drop_params=False` to the rejection check while the live chat path reads the deployment's drop_params. Both gates now read the same flag, so the knobs are dropped instead of the upload failing with a 400 The upload stream's `deployment_model` argument defaults to None again, so the two-argument callers (direct SDK uploads and the existing batch tests) keep working, and a row whose body.model is missing or not a string is still reported as a 400 naming the line when the deployment names no model --- .../llms/vertex_ai/files/transformation.py | 11 +++++--- .../test_vertex_ai_files_transformation.py | 25 ++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index d32341a498e..8ffaa5bf8e0 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -697,7 +697,9 @@ def _batch_row_model(deployment_model: str | None, row_model: object) -> str: binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied from an AI Studio call lands on its Vertex row instead of past every row.""" - chosen: Final = deployment_model if deployment_model else row_model if isinstance(row_model, str) else "" + chosen: Final = deployment_model if deployment_model else row_model + if not isinstance(chosen, str): + raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string") namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") return chosen.removeprefix(namespace) @@ -879,7 +881,7 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], - deployment_model: str | None, + deployment_model: str | None = None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params @@ -1143,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): sampling_params: Final = tuple( p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None ) + drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)) if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"): VertexGeminiConfig.reject_unsupported_sampling_param( - model=model, param=", ".join(sampling_params), drop_params=False + model=model, param=", ".join(sampling_params), drop_params=drop_params ) vertex_params: Final = config.map_openai_params( model=model, non_default_params=openai_request_body, optional_params={}, - drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)), + drop_params=drop_params, allow_integer_enums=False, ) return vertex_params diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index b4482e4a413..25ede9f0d01 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -2081,15 +2081,21 @@ def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_n ) -def _upload_rows(entries: Iterable[Mapping[str, object]], deployment_model: str | None) -> list[object]: +def _upload_rows( + entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None +) -> list[object]: """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's - `?model=` branch runs it, with the deployment's model in litellm_params""" + `?model=` branch runs it, with the deployment's model and drop_params in litellm_params""" jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + litellm_params: Final[dict[str, object]] = { + **({"model": deployment_model} if deployment_model else {}), + **({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}), + } request: Final = VertexAIFilesConfig().transform_create_file_request( model="", create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, optional_params={}, - litellm_params={"model": deployment_model} if deployment_model else {}, + litellm_params=litellm_params, ) assert isinstance(request, dict) body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) @@ -2119,6 +2125,19 @@ def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_par assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} +def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a + real-time call on that deployment does, instead of a 400 that only the global switch could avoid""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + (row,) = _upload_rows( + [_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True + ) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( monkeypatch: pytest.MonkeyPatch, ) -> None: From 1c9304a71873cb90f3f14d8766e9f0a03c5b48c0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 17:22:46 -0700 Subject: [PATCH 7/7] test(router): drop the vacuous supports_sampling_params case from the deployment isolation test supports_sampling_params became a cost-map schema field in #47645, so the router writes a deployment's value onto the shared backend row the same way it does supports_vision and friends. That shared row is the only place the Gemini sampling check reads the per-deployment escape hatch from, so the flag is not deployment-scoped like supports_regex_lookaround and supports_assistant_prefill are. The case only passed before because get_model_info never emitted the field, and this branch now surfaces it, so the parametrization is reduced to the flags that really stay on the deployment that set them --- tests/unit/test_router_model_cost_isolation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f001008ee2c..426481d2719 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -585,7 +585,7 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"]) +@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"]) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it."""