mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
feat(gemini)!: judge Gemini models the map has no sampling flag for by the version in their name
Google ignores temperature, top_p and top_k on Gemini 3.6 Flash, 3.5 Flash-Lite and every later model. The model map carries supports_sampling_params: false for the rows we know about, but a chat row without the flag, or a model newer than the map, kept forwarding the knobs silently. When no row answers, read the version out of the model name and apply the same drop-or-400 contract. A row whose mode is not chat keeps forwarding, since the notice covers the text models. gemini-4-argon has a chat row without the flag, so it now drops the knobs under drop_params and returns the 400 naming drop_params without it, like the flagged rows.
This commit is contained in:
parent
49b5444ef9
commit
7b8ab61415
2 changed files with 89 additions and 11 deletions
|
|
@ -429,23 +429,41 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _google_ignores_sampling_params_by_version(model_name: str) -> bool:
|
||||
"""Google's announced cut: Gemini 3.6 Flash, 3.5 Flash-Lite and every later model ignore the
|
||||
sampling knobs, so a name with such a version is judged by the version when the map has no answer."""
|
||||
match: Final = re.match(r"^gemini-(\d+)(?:\.(\d+))?(?:-|$)", model_name.lower())
|
||||
if match is None:
|
||||
return False
|
||||
version: Final = (int(match.group(1)), int(match.group(2) or 0))
|
||||
if version >= (3, 6):
|
||||
return True
|
||||
return version == (3, 5) and "flash-lite" in model_name.lower()
|
||||
|
||||
@staticmethod
|
||||
def supports_sampling_params(model: str) -> bool:
|
||||
"""Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``.
|
||||
|
||||
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
|
||||
and later accept the fields but ignore them, and a later generation will 400 on them.
|
||||
A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p``
|
||||
leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param
|
||||
check raises or drops them and ``allowed_openai_params`` still forwards them verbatim;
|
||||
``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the
|
||||
request body is built."""
|
||||
A chat model the map says nothing about is judged by the version in its name, so a model
|
||||
newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding.
|
||||
``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so
|
||||
the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still
|
||||
forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and
|
||||
is gated while the request body is built."""
|
||||
rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id
|
||||
bare: Final = model.split("/")[-1]
|
||||
for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"):
|
||||
if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool):
|
||||
matched: Final = tuple(
|
||||
row for key in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}") if (row := rows.get(key)) is not None
|
||||
)
|
||||
for row in matched:
|
||||
if isinstance(flag := row.get("supports_sampling_params"), bool):
|
||||
return flag
|
||||
return True
|
||||
if any(row.get("mode") not in (None, "chat") for row in matched):
|
||||
return True
|
||||
return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare)
|
||||
|
||||
@staticmethod
|
||||
def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None:
|
||||
|
|
|
|||
|
|
@ -3750,12 +3750,13 @@ def _gemini_request_body(
|
|||
custom_llm_provider: Literal["vertex_ai", "gemini"],
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
model: str = "gemini-zeta-9",
|
||||
) -> RequestBody:
|
||||
from litellm.llms.vertex_ai.gemini.transformation import transform_request_body
|
||||
|
||||
return transform_request_body(
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="gemini-zeta-9",
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -3902,6 +3903,65 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un
|
|||
assert optional_params["max_output_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"])
|
||||
@pytest.mark.parametrize(
|
||||
("model", "forwarded"),
|
||||
[
|
||||
("gemini-3.6-flash-zeta", False),
|
||||
("gemini-3.5-flash-lite-zeta", False),
|
||||
("gemini-3.7-pro-zeta", False),
|
||||
("gemini-4-zeta", False),
|
||||
("gemini-4.1-flash-zeta", False),
|
||||
("gemini-10-zeta", False),
|
||||
("gemini-3.5-flash-zeta", True),
|
||||
("gemini-3.5-pro-zeta", True),
|
||||
("gemini-3.1-pro-zeta", True),
|
||||
("gemini-2.5-flash-zeta", True),
|
||||
("gemini-zeta-9", True),
|
||||
],
|
||||
)
|
||||
def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name(
|
||||
custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool
|
||||
) -> None:
|
||||
"""Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer
|
||||
than the map gets the same drop-or-400 contract instead of silently forwarding (#47015 follow-up)"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
optional_params: Final = get_optional_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS
|
||||
)
|
||||
body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model)
|
||||
|
||||
knobs: Final = _SAMPLING_PARAMS if forwarded else {}
|
||||
assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("row", "forwarded"),
|
||||
[
|
||||
({"mode": "chat"}, False),
|
||||
({}, False),
|
||||
({"mode": "audio_speech"}, True),
|
||||
({"mode": "chat", "supports_sampling_params": True}, True),
|
||||
],
|
||||
ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"],
|
||||
)
|
||||
def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule(
|
||||
row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An explicit flag always wins, a row silent on it is judged by the version, and a row for another
|
||||
mode keeps forwarding because Google's notice covers the text models"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)
|
||||
|
||||
optional_params: Final = get_optional_params(
|
||||
model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True
|
||||
)
|
||||
|
||||
assert ("temperature" in optional_params) is forwarded
|
||||
|
||||
|
||||
def test_media_resolution_from_detail_parameter():
|
||||
"""Test that OpenAI's detail parameter is correctly mapped to media_resolution"""
|
||||
from litellm.llms.vertex_ai.gemini.transformation import (
|
||||
|
|
@ -6928,7 +6988,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl
|
|||
def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100))
|
||||
unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100))
|
||||
mapped: Final = _mapped_gemini_3_models()[0]
|
||||
config: Final = VertexGeminiConfig()
|
||||
|
||||
|
|
@ -6971,7 +7031,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_
|
|||
def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20))
|
||||
priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20))
|
||||
for name in priced:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue