mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(gemini): judge a batch deployment's backend row by the version rule too
The proxy registers a batch deployment's model_info onto the backend model's map row, so vertex_ai/gemini-4-argon carried mode: batch on a batch proxy and the version rule treated it as a non-chat row and kept forwarding the knobs. A live upload on the PR tip stored temperature, top_p and top_k in the bucket on both legs. Batch rows are chat requests, so a batch row is judged by the version like a chat row, and the batch transformation now has the regression
This commit is contained in:
parent
7b8ab61415
commit
c01d4acffa
3 changed files with 40 additions and 9 deletions
|
|
@ -448,7 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
|
||||
and later accept the fields but ignore them, and a later generation will 400 on them.
|
||||
A chat model the map says nothing about is judged by the version in its name, so a model
|
||||
newer than the map keeps the same contract; a non-chat row without the flag keeps forwarding.
|
||||
newer than the map keeps the same contract. A batch deployment's rows are chat requests and
|
||||
the proxy stamps its ``mode: batch`` onto the backend row, so that row is judged the same way;
|
||||
a row for any other mode without the flag keeps forwarding.
|
||||
``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so
|
||||
the shared unsupported-param check raises or drops them and ``allowed_openai_params`` still
|
||||
forwards them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and
|
||||
|
|
@ -461,7 +463,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
for row in matched:
|
||||
if isinstance(flag := row.get("supports_sampling_params"), bool):
|
||||
return flag
|
||||
if any(row.get("mode") not in (None, "chat") for row in matched):
|
||||
if any(row.get("mode") not in (None, "chat", "batch") for row in matched):
|
||||
return True
|
||||
return not VertexGeminiConfig._google_ignores_sampling_params_by_version(bare)
|
||||
|
||||
|
|
|
|||
|
|
@ -171,9 +171,7 @@ class TestBatchObjectNaming:
|
|||
def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config):
|
||||
"""A numeric endpoint id must not be filed under publishers/google/models/gemini/<id>,
|
||||
which the batch transformation later mangles into a nonexistent publisher model (LIT-6899)."""
|
||||
object_name = config._get_gcs_object_name_from_batch_jsonl(
|
||||
[{"body": {"model": "gemini/7768560373388541952"}}]
|
||||
)
|
||||
object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}])
|
||||
assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/")
|
||||
assert "publishers" not in object_name
|
||||
|
||||
|
|
@ -865,7 +863,6 @@ class TestVertexBatchOutputTransformation:
|
|||
assert config._try_transform_vertex_batch_output_to_openai(binary) == binary
|
||||
|
||||
|
||||
|
||||
class TestTryTransformDoesNotMutateCallerLoggingObj:
|
||||
"""Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate
|
||||
the caller's logging_obj (model, start_time, optional_params)."""
|
||||
|
|
@ -2045,3 +2042,27 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
|
|||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
|
||||
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
|
||||
|
||||
|
||||
def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"})
|
||||
|
||||
|
||||
def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini
|
||||
model whose name is past Google's cut must still be judged by its version and not pass as a non-chat row"""
|
||||
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"):
|
||||
_wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
|
||||
|
||||
|
||||
def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
(row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
|
||||
assert _generation_config(row) == {"max_output_tokens": 5}
|
||||
|
|
|
|||
|
|
@ -3941,16 +3941,24 @@ def test_gemini_model_unknown_to_the_map_is_judged_by_the_version_in_its_name(
|
|||
[
|
||||
({"mode": "chat"}, False),
|
||||
({}, False),
|
||||
({"mode": "batch"}, False),
|
||||
({"mode": "audio_speech"}, True),
|
||||
({"mode": "chat", "supports_sampling_params": True}, True),
|
||||
],
|
||||
ids=["chat_row_without_flag", "row_without_mode", "non_chat_row_without_flag", "explicit_true"],
|
||||
ids=[
|
||||
"chat_row_without_flag",
|
||||
"row_without_mode",
|
||||
"batch_deployment_row_without_flag",
|
||||
"non_chat_row_without_flag",
|
||||
"explicit_true",
|
||||
],
|
||||
)
|
||||
def test_map_row_for_a_gemini_4_model_decides_before_the_version_rule(
|
||||
row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""An explicit flag always wins, a row silent on it is judged by the version, and a row for another
|
||||
mode keeps forwarding because Google's notice covers the text models"""
|
||||
"""An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment
|
||||
is judged by the version, and a row for another mode keeps forwarding because Google's notice covers
|
||||
the text models"""
|
||||
from litellm.utils import get_optional_params
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue