From 7b8ab61415a23df02b697a8f6fa4396f02416eb0 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 16:52:52 -0700 Subject: [PATCH] feat(gemini)!: judge Gemini models the map has no sampling flag for by the version in their name Google ignores temperature, top_p and top_k on Gemini 3.6 Flash, 3.5 Flash-Lite and every later model. The model map carries supports_sampling_params: false for the rows we know about, but a chat row without the flag, or a model newer than the map, kept forwarding the knobs silently. When no row answers, read the version out of the model name and apply the same drop-or-400 contract. A row whose mode is not chat keeps forwarding, since the notice covers the text models. gemini-4-argon has a chat row without the flag, so it now drops the knobs under drop_params and returns the 400 naming drop_params without it, like the flagged rows. --- .../vertex_and_google_ai_studio_gemini.py | 34 +++++++--- ...test_vertex_and_google_ai_studio_gemini.py | 66 ++++++++++++++++++- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 241b2d2a3a0..f2491a7a490 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -429,23 +429,41 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False return True + @staticmethod + def _google_ignores_sampling_params_by_version(model_name: str) -> bool: + """Google's announced cut: Gemini 3.6 Flash, 3.5 Flash-Lite and every later model ignore the + sampling knobs, so a name with such a version is judged by the version when the map has no answer.""" + match: Final = re.match(r"^gemini-(\d+)(?:\.(\d+))?(?:-|$)", model_name.lower()) + if match is None: + return False + version: Final = (int(match.group(1)), int(match.group(2) or 0)) + if version >= (3, 6): + return True + return version == (3, 5) and "flash-lite" in model_name.lower() + @staticmethod def supports_sampling_params(model: str) -> bool: """Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``. Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite and later accept the fields but ignore them, and a later generation will 400 on them. - A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p`` - leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param - check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; - ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the - request body is built.""" + A chat model the map says nothing about is judged by the version in its name, so a model + newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding. + ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so + the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still + forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and + is gated while the request body is built.""" rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id bare: Final = model.split("/")[-1] - for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"): - if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool): + matched: Final = tuple( + row for key in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}") if (row := rows.get(key)) is not None + ) + for row in matched: + if isinstance(flag := row.get("supports_sampling_params"), bool): return flag - return True + if any(row.get("mode") not in (None, "chat") for row in matched): + return True + return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare) @staticmethod def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None: diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bb624add673..8b5787006be 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3750,12 +3750,13 @@ def _gemini_request_body( custom_llm_provider: Literal["vertex_ai", "gemini"], optional_params: dict[str, object], litellm_params: dict[str, object], + model: str = "gemini-zeta-9", ) -> RequestBody: from litellm.llms.vertex_ai.gemini.transformation import transform_request_body return transform_request_body( messages=[{"role": "user", "content": "hi"}], - model="gemini-zeta-9", + model=model, optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, @@ -3902,6 +3903,65 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un assert optional_params["max_output_tokens"] == 10 +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +@pytest.mark.parametrize( + ("model", "forwarded"), + [ + ("gemini-3.6-flash-zeta", False), + ("gemini-3.5-flash-lite-zeta", False), + ("gemini-3.7-pro-zeta", False), + ("gemini-4-zeta", False), + ("gemini-4.1-flash-zeta", False), + ("gemini-10-zeta", False), + ("gemini-3.5-flash-zeta", True), + ("gemini-3.5-pro-zeta", True), + ("gemini-3.1-pro-zeta", True), + ("gemini-2.5-flash-zeta", True), + ("gemini-zeta-9", True), + ], +) +def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name( + custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool +) -> None: + """Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer + than the map gets the same drop-or-400 contract instead of silently forwarding (#47015 follow-up)""" + from litellm.utils import get_optional_params + + optional_params: Final = get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS + ) + body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model) + + knobs: Final = _SAMPLING_PARAMS if forwarded else {} + assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model + + +@pytest.mark.parametrize( + ("row", "forwarded"), + [ + ({"mode": "chat"}, False), + ({}, False), + ({"mode": "audio_speech"}, True), + ({"mode": "chat", "supports_sampling_params": True}, True), + ], + ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"], +) +def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule( + row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """An explicit flag always wins, a row silent on it is judged by the version, and a row for another + mode keeps forwarding because Google's notice covers the text models""" + from litellm.utils import get_optional_params + + monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row) + + optional_params: Final = get_optional_params( + model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True + ) + + assert ("temperature" in optional_params) is forwarded + + def test_media_resolution_from_detail_parameter(): """Test that OpenAI's detail parameter is correctly mapped to media_resolution""" from litellm.llms.vertex_ai.gemini.transformation import ( @@ -6928,7 +6988,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( caplog: pytest.LogCaptureFixture, ) -> None: - unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100)) mapped: Final = _mapped_gemini_3_models()[0] config: Final = VertexGeminiConfig() @@ -6971,7 +7031,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20)) for name in priced: monkeypatch.setitem( litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed