fix(gemini): judge a batch deployment's backend row by the version rule too

The proxy registers a batch deployment's model_info onto the backend model's
map row, so vertex_ai/gemini-4-argon carried mode: batch on a batch proxy and
the version rule treated it as a non-chat row and kept forwarding the knobs.
A live upload on the PR tip stored temperature, top_p and top_k in the bucket
on both legs. Batch rows are chat requests, so a batch row is judged by the
version like a chat row, and the batch transformation now has the regression
This commit is contained in:
ryan-crabbe-berri 2026-10-20 17:14:30 -07:00
parent 7b8ab61415
commit c01d4acffa
3 changed files with 40 additions and 9 deletions

View file

@ -448,7 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
and later accept the fields but ignore them, and a later generation will 400 on them.
A chat model the map says nothing about is judged by the version in its name, so a model
newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding.
newer than the map keeps the same contract. A batch deployment's rows are chat requests and
the proxy stamps its ``mode: batch`` onto the backend row, so that row is judged the same way;
a row for any other mode without the flag keeps forwarding.
``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so
the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still
forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and
@ -461,7 +463,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
for row in matched:
if isinstance(flag := row.get("supports_sampling_params"), bool):
return flag
if any(row.get("mode") not in (None, "chat") for row in matched):
if any(row.get("mode") not in (None, "chat", "batch") for row in matched):
return True
return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare)

View file

@ -171,9 +171,7 @@ class TestBatchObjectNaming:
def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config):
"""A numeric endpoint id must not be filed under publishers/google/models/gemini/<id>,
which the batch transformation later mangles into a nonexistent publisher model (LIT-6899)."""
object_name = config._get_gcs_object_name_from_batch_jsonl(
[{"body": {"model": "gemini/7768560373388541952"}}]
)
object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}])
assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/")
assert "publishers" not in object_name
@ -865,7 +863,6 @@ class TestVertexBatchOutputTransformation:
assert config._try_transform_vertex_batch_output_to_openai(binary) == binary
class TestTryTransformDoesNotMutateCallerLoggingObj:
"""Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate
the caller's logging_obj (model, start_time, optional_params)."""
@ -2045,3 +2042,27 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
monkeypatch.setattr(litellm, "drop_params", False)
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"})
def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini
model whose name is past Google's cut must still be judged by its version and not pass as a non-chat row"""
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"):
_wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
monkeypatch.setattr(litellm, "drop_params", True)
(row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5}

View file

@ -3941,16 +3941,24 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name(
[
({"mode": "chat"}, False),
({}, False),
({"mode": "batch"}, False),
({"mode": "audio_speech"}, True),
({"mode": "chat", "supports_sampling_params": True}, True),
],
ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"],
ids=[
"chat_row_without_flag",
"row_without_mode",
"batch_deployment_row_without_flag",
"non_chat_row_without_flag",
"explicit_true",
],
)
def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule(
row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
"""An explicit flag always wins, a row silent on it is judged by the version, and a row for another
mode keeps forwarding because Google's notice covers the text models"""
"""An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment
is judged by the version, and a row for another mode keeps forwarding because Google's notice covers
the text models"""
from litellm.utils import get_optional_params
monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)