This commit is contained in:
ryan-crabbe-berri 2026-10-27 00:26:12 +00:00 • committed by GitHub
commit eed07b47e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 371 additions and 52 deletions

View file

@ -81,8 +81,14 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
def is_model_gemini_audio_model(self, model: str) -> bool:
return "tts" in model
@property
def custom_llm_provider(self) -> str:
return "gemini"
def get_supported_openai_params(self, model: str) -> list[str]:
sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else ()
sampling_params: Final = (
("temperature", "top_p") if self.supports_sampling_params(model, self.custom_llm_provider) else ()
)
supported_params: Final = [
*sampling_params,
"max_tokens",

View file

@ -684,9 +684,30 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
)
_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/")
def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None:
configured_model: Final = litellm_params.get("model")
return configured_model if isinstance(configured_model, str) else None
def _batch_row_model(deployment_model: str | None, row_model: object) -> str:
"""The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path
binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google
namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied
from an AI Studio call lands on its Vertex row instead of past every row."""
chosen: Final = deployment_model if deployment_model else row_model
if not isinstance(chosen, str):
raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string")
namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "")
return chosen.removeprefix(namespace)
def _openai_batch_jsonl_entry_to_vertex_rows(
openai_entry: dict[str, Any],
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
deployment_model: str | None = None,
) -> tuple[Mapping[str, object], ...]:
"""
Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
@ -706,10 +727,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
if _is_responses_batch_entry(openai_entry)
else openai_request_body
)
row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))}
vertex_request_body: Final = transform_request_body(
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
model=chat_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(chat_request_body),
messages=map_developer_role_to_system_role(row_body.get("messages", [])),
model=row_body["model"],
optional_params=map_openai_to_vertex_params(row_body),
custom_llm_provider="vertex_ai",
litellm_params={},
cached_content=None,
@ -859,15 +881,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
self,
openai_file_content: FileTypes,
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
deployment_model: str | None = None,
) -> None:
self._openai_file_content = openai_file_content
self._map_openai_to_vertex_params = map_openai_to_vertex_params
self._deployment_model = deployment_model
def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
first = True
for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
try:
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
entry, self._map_openai_to_vertex_params, self._deployment_model
)
except Exception as e:
raise VertexAIError(
status_code=400,
@ -1053,7 +1079,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
data: CreateFileRequest,
) -> str:
"""
@ -1077,11 +1103,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
if purpose is None:
raise ValueError("purpose is required")
configured_model: Final = litellm_params.get("model")
object_name = self.get_object_name(
file_data,
purpose,
deployment_model=configured_model if isinstance(configured_model, str) else None,
deployment_model=_configured_deployment_model(litellm_params),
passthrough=is_passthrough_batch_upload(data, litellm_params),
)
if object_prefix:
@ -1120,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
sampling_params: Final = tuple(
p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None
)
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model):
drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params))
if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"):
VertexGeminiConfig.reject_unsupported_sampling_param(
model=model, param=", ".join(sampling_params), drop_params=False
model=model, param=", ".join(sampling_params), drop_params=drop_params
)
vertex_params: Final = config.map_openai_params(
model=model,
non_default_params=openai_request_body,
optional_params={},
drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)),
drop_params=drop_params,
allow_integer_enums=False,
)
return vertex_params
@ -1138,7 +1164,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
model: str,
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
) -> bytes | str | dict:
"""
2 Cases:
@ -1170,6 +1196,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
body_stream=_OpenAIToVertexBatchUploadStream(
file_data,
partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
_configured_deployment_model(litellm_params),
),
content_type="application/json",
)

View file

@ -1200,7 +1200,9 @@ def transform_request_body(
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model)
drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(
model, custom_llm_provider
)
if drops_top_k:
request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature
VertexGeminiConfig.reject_unsupported_sampling_param(

View file

@ -702,23 +702,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return False
return True
@property
def custom_llm_provider(self) -> str:
return "vertex_ai"
@staticmethod
def supports_sampling_params(model: str) -> bool:
def supports_sampling_params(model: str, custom_llm_provider: str | None) -> bool:
"""Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``.
Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite
and later accept the fields but ignore them, and a later generation will 400 on them.
A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p``
leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param
check raises or drops them and ``allowed_openai_params`` still forwards them verbatim;
``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the
request body is built."""
rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id
bare: Final = model.split("/")[-1]
for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"):
if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool):
return flag
return True
Read from the model map through the provider-aware lookup: an explicit ``supports_sampling_params``
on the row wins, a chat, Responses or batch row without one and an id the map has never seen are
answered by the ``gemini-sampling-params-ignored`` fallback generalization (Gemini 3.6 Flash, 3.5
Flash-Lite and every later text model), and a row for any other mode keeps forwarding.
``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the
shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards
them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while
the request body is built."""
return not is_explicitly_disabled_factory(model, custom_llm_provider, "supports_sampling_params")
@staticmethod
def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None:
@ -728,15 +728,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return
raise UnsupportedParamsError(
message=(
f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` "
"because Google ignores sampling params on this model. "
"To drop unsupported params, set `litellm.drop_params = True`."
f"{model} does not support {param}: Google ignores sampling params on this model, so LiteLLM "
"treats it as `supports_sampling_params: false`. To drop unsupported params, set "
"`litellm.drop_params = True`, or set `supports_sampling_params: true` in the model's "
"model_info to forward them anyway."
),
status_code=400,
)
def get_supported_openai_params(self, model: str) -> list[str]:
sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else ()
sampling_params: Final = (
("temperature", "top_p", "top_k") if self.supports_sampling_params(model, self.custom_llm_provider) else ()
)
supported_params: Final = [
*sampling_params,
"max_tokens",
@ -1616,7 +1619,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
allow_integer_enums: bool = True,
) -> dict:
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
supports_sampling_params: Final = self.supports_sampling_params(model)
supports_sampling_params: Final = self.supports_sampling_params(model, self.custom_llm_provider)
for param, value in non_default_params.items():
if param in ("temperature", "top_p", "top_k") and not supports_sampling_params:
continue

View file

@ -61322,6 +61322,15 @@
"supports_prompt_caching": true,
"supports_web_search": true
}
},
{
"name": "gemini-sampling-params-ignored",
"pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)",
"fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"],
"description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini-<major>-<minor>) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.",
"model_info": {
"supports_sampling_params": false
}
}
]
},

View file

@ -277,7 +277,7 @@ from litellm.types.utils import (
)
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
_BACKFILL_MODES: Final = frozenset({"chat", "responses", "batch"})
# +-----------------------------------------------+
# | |
@ -6697,7 +6697,9 @@ def get_model_info_helper(
_model_info: dict[str, Any] | None = lookup[1]
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
fill_missing: Final = match_fill_missing_generalizations(
key, _model_info.get("litellm_provider") or model_cost_custom_llm_provider or ""
)
if fill_missing is not None:
_model_info = {
**{k: v for k, v in fill_missing.items() if k not in _model_info},
@ -6987,6 +6989,7 @@ def get_model_info_helper(
supports_web_search=_model_info.get("supports_web_search", None),
supports_url_context=_model_info.get("supports_url_context", None),
supports_reasoning=_model_info.get("supports_reasoning", None),
supports_sampling_params=_model_info.get("supports_sampling_params", None),
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
thinking_always_on=_model_info.get("thinking_always_on", None),

View file

@ -61322,6 +61322,15 @@
"supports_prompt_caching": true,
"supports_web_search": true
}
},
{
"name": "gemini-sampling-params-ignored",
"pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)",
"fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"],
"description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini-<major>-<minor>) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.",
"model_info": {
"supports_sampling_params": false
}
}
]
},

View file

@ -390,6 +390,17 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
"litellm_provider": "openrouter",
"mode": "chat",
},
"acme-batch": {
"input_cost_per_token": 9e-6,
"output_cost_per_token": 1e-5,
"litellm_provider": "openai",
"mode": "batch",
},
"acme-unowned": {
"input_cost_per_token": 2e-6,
"output_cost_per_token": 3e-6,
"mode": "chat",
},
},
)
restore_generalizations(
@ -422,6 +433,14 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp
image = litellm.get_model_info("acme-image", custom_llm_provider="openai")
assert image.get("supports_reasoning") is None
batch = litellm.get_model_info("acme-batch", custom_llm_provider="openai")
assert batch["supports_reasoning"] is True
assert batch["max_tokens"] == 5
unowned = litellm.get_model_info("acme-unowned", custom_llm_provider="openai")
assert unowned["supports_reasoning"] is True
assert litellm.get_model_info("acme-unowned", custom_llm_provider="openrouter").get("supports_reasoning") is None
restore_generalizations(
[{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}]
)
@ -506,7 +525,7 @@ def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cos
"databricks-gemini-3-1-flash-image",
"openrouter/google/gemini-2.0-flash-001",
):
assert match_capability_generalizations(model) is None, model
assert "mode" not in (match_capability_generalizations(model) or {}), model
def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map):
@ -985,3 +1004,60 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None
@pytest.mark.parametrize(
"model,provider,sampling",
[
("gemini-3.6-flash-zeta", "gemini", False),
("gemini-3.5-flash-lite-zeta", "vertex_ai", False),
("gemini-3.10-pro-zeta", "gemini", False),
("gemini-4-zeta", "gemini", False),
("gemini-4.1-flash-zeta", "vertex_ai", False),
("gemini-10-zeta", "gemini", False),
("vertex_ai/gemini-4-zeta", None, False),
("gemini-3.5-flash-zeta", "gemini", None),
("gemini-3.5-pro-zeta", "vertex_ai", None),
("gemini-3.1-pro-zeta", "gemini", None),
("gemini-2.5-flash-zeta", "gemini", None),
],
)
def test_shipped_gemini_sampling_rule_version_boundaries(shipped_cost_map, model, provider, sampling):
"""The gemini-sampling-params-ignored rule marks Gemini 3.6 and every later 3.x minor, every 4+ major
and 3.5 Flash-Lite as not honoring temperature, top_p and top_k, and leaves 3.5 Flash, 3.5 Pro, 3.1 and
2.5 ids without an opinion"""
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info.get("supports_sampling_params") is sampling, model
def test_shipped_gemini_sampling_rule_skips_non_text_ids(shipped_cost_map):
for model in (
"gemini/gemini-4-flash-image",
"gemini/gemini-3.8-flash-tts",
"gemini/gemini-4-flash-live-preview",
"gemini/gemini-3.6-flash-native-audio",
"gemini/gemini-embedding-4",
"gemini/gemini-4-flash-transcribe",
"gemini/gemini-3.6-flash-live-translate-preview",
"gemini/gemini-4-computer-use-preview",
"gemini/gemini-zeta-9",
):
assert (match_capability_generalizations(model) or {}).get("supports_sampling_params") is None, model
def test_shipped_gemini_sampling_rule_fills_mapped_google_rows_without_the_flag(shipped_cost_map):
"""A mapped Gemini row past Google's cut reads as supports_sampling_params: false on AI Studio and Vertex
whether or not someone flagged it by hand, and a reseller copy of the same model is left alone"""
for key, model, provider in (
("gemini/gemini-4-argon", "gemini-4-argon", "gemini"),
("vertex_ai/gemini-4-argon", "gemini-4-argon", "vertex_ai"),
("gemini/gemini-3.6-flash", "gemini-3.6-flash", "gemini"),
):
assert key in litellm.model_cost
assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_sampling_params"] is False, key
assert match_fill_missing_generalizations("gemini-4-argon", "vertex_ai-language-models") == {
"supports_sampling_params": False
}
assert match_fill_missing_generalizations("gemini-4-argon", "github_copilot") is None

View file

@ -5,25 +5,26 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
import json
import urllib.parse
from collections.abc import Mapping
from typing import Final
from collections.abc import Iterable, Mapping
from types import MappingProxyType
from typing import Final
from unittest.mock import MagicMock
from urllib.parse import parse_qs, urlparse
import httpx
import pytest
from openai.types.file_deleted import FileDeleted
from pydantic import TypeAdapter
from unittest.mock import MagicMock
import litellm
from litellm.llms.vertex_ai.common_utils import VertexAIError
from litellm.llms.vertex_ai.files.transformation import (
VertexAIFilesConfig,
_get_litellm_batch_custom_id_from_labels,
_openai_batch_jsonl_entry_to_vertex_rows,
_sanitize_gcp_label_value,
)
from litellm.types.llms.openai import OpenAIFileObject, HttpxBinaryResponseContent
from openai.types.file_deleted import FileDeleted
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
@pytest.fixture
@ -171,9 +172,7 @@ class TestBatchObjectNaming:
def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config):
"""A numeric endpoint id must not be filed under publishers/google/models/gemini/<id>,
which the batch transformation later mangles into a nonexistent publisher model (LIT-6899)."""
object_name = config._get_gcs_object_name_from_batch_jsonl(
[{"body": {"model": "gemini/7768560373388541952"}}]
)
object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}])
assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/")
assert "publishers" not in object_name
@ -865,7 +864,6 @@ class TestVertexBatchOutputTransformation:
assert config._try_transform_vertex_batch_output_to_openai(binary) == binary
class TestTryTransformDoesNotMutateCallerLoggingObj:
"""Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate
the caller's logging_obj (model, start_time, optional_params)."""
@ -2045,3 +2043,110 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
monkeypatch.setattr(litellm, "drop_params", False)
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"})
@pytest.mark.usefixtures("local_model_cost_map")
def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini
model whose name is past Google's cut must still be filled by the shipped sampling rule and not pass as a
non-chat row"""
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"):
_wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
@pytest.mark.usefixtures("local_model_cost_map")
def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta")
monkeypatch.setattr(litellm, "drop_params", True)
(row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5}
def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None:
monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"})
monkeypatch.setitem(
litellm.model_cost,
f"vertex_ai/{model_name}",
{"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False},
)
def _upload_rows(
entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None
) -> list[object]:
"""The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's
`?model=<deployment>` branch runs it, with the deployment's model and drop_params in litellm_params"""
jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode()
litellm_params: Final[dict[str, object]] = {
**({"model": deployment_model} if deployment_model else {}),
**({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}),
}
request: Final = VertexAIFilesConfig().transform_create_file_request(
model="",
create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"},
optional_params={},
litellm_params=litellm_params,
)
assert isinstance(request, dict)
body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes())
return [json.loads(line) for line in body.decode().splitlines()]
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row(
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag
decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were
stored even though the model ignores them"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"):
_upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"])
def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params(
deployment_model: str | None, monkeypatch: pytest.MonkeyPatch
) -> None:
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", True)
(row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model)
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a
real-time call on that deployment does, instead of a 400 that only the global switch could avoid"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setattr(litellm, "drop_params", False)
(row,) = _upload_rows(
[_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True
)
assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5}
def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that
still takes the knobs is gated for the model that actually executes it"""
_google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9")
monkeypatch.setitem(
litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"}
)
monkeypatch.setattr(litellm, "drop_params", False)
with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"):
_upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9")

View file

@ -4116,12 +4116,13 @@ def _gemini_request_body(
custom_llm_provider: Literal["vertex_ai", "gemini"],
optional_params: dict[str, object],
litellm_params: dict[str, object],
model: str = "gemini-zeta-9",
) -> RequestBody:
from litellm.llms.vertex_ai.gemini.transformation import transform_request_body
return transform_request_body(
messages=[{"role": "user", "content": "hi"}],
model="gemini-zeta-9",
model=model,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
@ -4217,13 +4218,21 @@ def test_gemini_keeps_forwarding_sampling_params_unless_the_model_map_flags_them
assert body["generationConfig"] == _SAMPLING_PARAMS
@pytest.mark.parametrize("model_key", ["gemini-zeta-9", "vertex_ai/gemini-zeta-9", "gemini/gemini-zeta-9"])
def test_gemini_sampling_flag_is_read_from_whichever_provider_row_carries_it(
model_key: str, monkeypatch: pytest.MonkeyPatch
@pytest.mark.parametrize(
("config_cls", "model_key"),
[
("vertex", "gemini-zeta-9"),
("vertex", "vertex_ai/gemini-zeta-9"),
("studio", "gemini-zeta-9"),
("studio", "gemini/gemini-zeta-9"),
],
)
def test_gemini_sampling_flag_is_read_from_the_row_of_the_route_serving_the_call(
config_cls: Literal["vertex", "studio"], model_key: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setitem(litellm.model_cost, model_key, {"supports_sampling_params": False})
result: Final = VertexGeminiConfig().map_openai_params(
result: Final = _gemini_config(config_cls).map_openai_params(
non_default_params={"temperature": 0.3},
optional_params={},
model="gemini-zeta-9",
@ -4268,6 +4277,76 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un
assert optional_params["max_output_tokens"] == 10
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"])
@pytest.mark.parametrize(
("model", "forwarded"),
[
("gemini-3.6-flash-zeta", False),
("gemini-3.5-flash-lite-zeta", False),
("gemini-3.7-pro-zeta", False),
("gemini-4-zeta", False),
("gemini-4.1-flash-zeta", False),
("gemini-10-zeta", False),
("gemini-3.5-flash-zeta", True),
("gemini-3.5-pro-zeta", True),
("gemini-3.1-pro-zeta", True),
("gemini-2.5-flash-zeta", True),
("gemini-zeta-9", True),
],
)
def test_gemini_model_unknown_to_the_map_is_judged_by_the_shipped_sampling_rule(
custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool
) -> None:
"""Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer
than the map gets the same drop-or-400 contract from the gemini-sampling-params-ignored fallback
generalization instead of silently forwarding (#47015 follow-up)"""
from litellm.utils import get_optional_params
optional_params: Final = get_optional_params(
model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS
)
body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model)
knobs: Final = _SAMPLING_PARAMS if forwarded else {}
assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize(
("row", "forwarded"),
[
({"mode": "chat"}, False),
({}, False),
({"mode": "batch"}, False),
({"mode": "audio_speech"}, True),
({"mode": "chat", "supports_sampling_params": True}, True),
],
ids=[
"chat_row_without_flag",
"row_without_mode",
"batch_deployment_row_without_flag",
"non_chat_row_without_flag",
"explicit_true",
],
)
def test_map_row_for_a_gemini_4_model_decides_before_the_shipped_sampling_rule(
row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
"""An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment
is filled by the shipped rule, and a row for another mode keeps forwarding because Google's notice covers
the text models"""
from litellm.utils import get_optional_params
monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row)
optional_params: Final = get_optional_params(
model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True
)
assert ("temperature" in optional_params) is forwarded
def test_media_resolution_from_detail_parameter():
"""Test that OpenAI's detail parameter is correctly mapped to media_resolution"""
from litellm.llms.vertex_ai.gemini.transformation import (
@ -7308,7 +7387,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl
def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models(
caplog: pytest.LogCaptureFixture,
) -> None:
unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100))
unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100))
mapped: Final = _mapped_gemini_3_models()[0]
config: Final = VertexGeminiConfig()
@ -7351,7 +7430,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_
def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20))
priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20))
for name in priced:
monkeypatch.setitem(
litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed

View file

@ -585,7 +585,7 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing():
)
@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"])
@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"])
def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None:
"""A deployment's capability override must not land on the shared
``{provider}/{model}`` key, or every sibling deployment of that model would inherit it."""