diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 936919af83d..9d757745a6d 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -81,8 +81,14 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): def is_model_gemini_audio_model(self, model: str) -> bool: return "tts" in model + @property + def custom_llm_provider(self) -> str: + return "gemini" + def get_supported_openai_params(self, model: str) -> list[str]: - sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else () + sampling_params: Final = ( + ("temperature", "top_p") if self.supports_sampling_params(model, self.custom_llm_provider) else () + ) supported_params: Final = [ *sampling_params, "max_tokens", diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 410f29ef88a..8ffaa5bf8e0 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -684,9 +684,30 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_GOOGLE_MODEL_NAMESPACES: Final = ("vertex_ai/", "gemini/") + + +def _configured_deployment_model(litellm_params: Mapping[str, object]) -> str | None: + configured_model: Final = litellm_params.get("model") + return configured_model if isinstance(configured_model, str) else None + + +def _batch_row_model(deployment_model: str | None, row_model: object) -> str: + """The Gemini id a batch row is shaped and gated for: the deployment's model, which the stored GCS path + binds the job to, else the row's own id for a direct SDK upload that names no deployment. The Google + namespace comes off either way, as ``get_llm_provider`` does for a live call, so a ``gemini/`` id copied + from an AI Studio call lands on its Vertex row instead of past every row.""" + chosen: Final = deployment_model if deployment_model else row_model + if not isinstance(chosen, str): + raise TypeError("`body.model` on batch requests must be a string, but was missing or not a string") + namespace: Final = next((ns for ns in _GOOGLE_MODEL_NAMESPACES if chosen.startswith(ns)), "") + return chosen.removeprefix(namespace) + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None = None, ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. @@ -706,10 +727,11 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + row_body: Final = {**chat_request_body, "model": _batch_row_model(deployment_model, chat_request_body.get("model"))} vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), - model=chat_request_body.get("model", ""), - optional_params=map_openai_to_vertex_params(chat_request_body), + messages=map_developer_role_to_system_role(row_body.get("messages", [])), + model=row_body["model"], + optional_params=map_openai_to_vertex_params(row_body), custom_llm_provider="vertex_ai", litellm_params={}, cached_content=None, @@ -859,15 +881,19 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + deployment_model: str | None = None, ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._deployment_model = deployment_model def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, self._map_openai_to_vertex_params, self._deployment_model + ) except Exception as e: raise VertexAIError( status_code=400, @@ -1053,7 +1079,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], data: CreateFileRequest, ) -> str: """ @@ -1077,11 +1103,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_configured_deployment_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1120,15 +1145,16 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): sampling_params: Final = tuple( p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None ) - if sampling_params and not VertexGeminiConfig.supports_sampling_params(model): + drop_params: Final = drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)) + if sampling_params and not VertexGeminiConfig.supports_sampling_params(model, "vertex_ai"): VertexGeminiConfig.reject_unsupported_sampling_param( - model=model, param=", ".join(sampling_params), drop_params=False + model=model, param=", ".join(sampling_params), drop_params=drop_params ) vertex_params: Final = config.map_openai_params( model=model, non_default_params=openai_request_body, optional_params={}, - drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)), + drop_params=drop_params, allow_integer_enums=False, ) return vertex_params @@ -1138,7 +1164,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1170,6 +1196,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params), + _configured_deployment_model(litellm_params), ), content_type="application/json", ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index ed86934db63..56d471ea803 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -1200,7 +1200,9 @@ def transform_request_body( from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig - drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model) + drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params( + model, custom_llm_provider + ) if drops_top_k: request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature VertexGeminiConfig.reject_unsupported_sampling_param( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 9acbc9c459c..b06eb76cfab 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -702,23 +702,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False return True + @property + def custom_llm_provider(self) -> str: + return "vertex_ai" + @staticmethod - def supports_sampling_params(model: str) -> bool: + def supports_sampling_params(model: str, custom_llm_provider: str | None) -> bool: """Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``. - Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite - and later accept the fields but ignore them, and a later generation will 400 on them. - A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p`` - leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param - check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; - ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the - request body is built.""" - rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id - bare: Final = model.split("/")[-1] - for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"): - if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool): - return flag - return True + Read from the model map through the provider-aware lookup: an explicit ``supports_sampling_params`` + on the row wins, a chat, Responses or batch row without one and an id the map has never seen are + answered by the ``gemini-sampling-params-ignored`` fallback generalization (Gemini 3.6 Flash, 3.5 + Flash-Lite and every later text model), and a row for any other mode keeps forwarding. + ``temperature`` and ``top_p`` leave ``get_supported_openai_params`` when the flag is off, so the + shared unsupported-param check raises or drops them and ``allowed_openai_params`` still forwards + them verbatim; ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while + the request body is built.""" + return not is_explicitly_disabled_factory(model, custom_llm_provider, "supports_sampling_params") @staticmethod def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None: @@ -728,15 +728,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return raise UnsupportedParamsError( message=( - f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` " - "because Google ignores sampling params on this model. " - "To drop unsupported params, set `litellm.drop_params = True`." + f"{model} does not support {param}: Google ignores sampling params on this model, so LiteLLM " + "treats it as `supports_sampling_params: false`. To drop unsupported params, set " + "`litellm.drop_params = True`, or set `supports_sampling_params: true` in the model's " + "model_info to forward them anyway." ), status_code=400, ) def get_supported_openai_params(self, model: str) -> list[str]: - sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else () + sampling_params: Final = ( + ("temperature", "top_p", "top_k") if self.supports_sampling_params(model, self.custom_llm_provider) else () + ) supported_params: Final = [ *sampling_params, "max_tokens", @@ -1616,7 +1619,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) - supports_sampling_params: Final = self.supports_sampling_params(model) + supports_sampling_params: Final = self.supports_sampling_params(model, self.custom_llm_provider) for param, value in non_default_params.items(): if param in ("temperature", "top_p", "top_k") and not supports_sampling_params: continue diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c913c879ccf..a5b3ed32352 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -61322,6 +61322,15 @@ "supports_prompt_caching": true, "supports_web_search": true } + }, + { + "name": "gemini-sampling-params-ignored", + "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)", + "fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"], + "description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini--) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.", + "model_info": { + "supports_sampling_params": false + } } ] }, diff --git a/litellm/utils.py b/litellm/utils.py index 1aed15ba83e..27ae94e77bd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -277,7 +277,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} -_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) +_BACKFILL_MODES: Final = frozenset({"chat", "responses", "batch"}) # +-----------------------------------------------+ # | | @@ -6697,7 +6697,9 @@ def get_model_info_helper( _model_info: dict[str, Any] | None = lookup[1] if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", "")) + fill_missing: Final = match_fill_missing_generalizations( + key, _model_info.get("litellm_provider") or model_cost_custom_llm_provider or "" + ) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, @@ -6987,6 +6989,7 @@ def get_model_info_helper( supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), + supports_sampling_params=_model_info.get("supports_sampling_params", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None), thinking_always_on=_model_info.get("thinking_always_on", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c913c879ccf..a5b3ed32352 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -61322,6 +61322,15 @@ "supports_prompt_caching": true, "supports_web_search": true } + }, + { + "name": "gemini-sampling-params-ignored", + "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:3[.-](?:[6-9]|[1-9]\\d)(?!\\d)|(?:[4-9]|[1-9]\\d)(?:[.-]\\d{1,2})?(?!\\d)|3[.-]5-flash-lite)(?:-|$)", + "fill_missing_for_providers": ["gemini", "vertex_ai", "vertex_ai-language-models"], + "description": "Any Gemini text id at or past the version where Google stopped honoring temperature, top_p and top_k: 3.6 and every later 3.x minor, every major from 4 up with or without a minor, and 3.5 Flash-Lite by name, under any namespace (bare, gemini/, vertex_ai/, databricks-gemini--) and with any trailing variant tag, so gemini-4-argon and gemini-3.6-flash-preview match while gemini-3.5-flash, gemini-3.1-pro and the 2.x line do not. The same lookahead as gemini-chat-baseline keeps the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines out, since Google's notice covers the text models. Google announced with Gemini 3.6 Flash and 3.5 Flash-Lite that these models accept the sampling fields but ignore them and that a later generation will reject them (reported in #47015, 2026-10), so an unmapped id past the cut resolves to supports_sampling_params: false and a mapped Google row the map has not flagged yet is filled the same way; the Gemini and Vertex transformations turn that into a drop under drop_params or a 400 that names it. The fill is limited to Google's own providers because resellers route through transformations that do not read the flag. An explicit supports_sampling_params on a row, including one a deployment's model_info registers, always wins.", + "model_info": { + "supports_sampling_params": false + } } ] }, diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index e59252e1571..2ff903de553 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -390,6 +390,17 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "litellm_provider": "openrouter", "mode": "chat", }, + "acme-batch": { + "input_cost_per_token": 9e-6, + "output_cost_per_token": 1e-5, + "litellm_provider": "openai", + "mode": "batch", + }, + "acme-unowned": { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 3e-6, + "mode": "chat", + }, }, ) restore_generalizations( @@ -422,6 +433,14 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp image = litellm.get_model_info("acme-image", custom_llm_provider="openai") assert image.get("supports_reasoning") is None + batch = litellm.get_model_info("acme-batch", custom_llm_provider="openai") + assert batch["supports_reasoning"] is True + assert batch["max_tokens"] == 5 + + unowned = litellm.get_model_info("acme-unowned", custom_llm_provider="openai") + assert unowned["supports_reasoning"] is True + assert litellm.get_model_info("acme-unowned", custom_llm_provider="openrouter").get("supports_reasoning") is None + restore_generalizations( [{"name": "acme-backfill", "pattern": r"^acme-", "model_info": {"supports_reasoning": True, "max_tokens": 5}}] ) @@ -506,7 +525,7 @@ def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cos "databricks-gemini-3-1-flash-image", "openrouter/google/gemini-2.0-flash-001", ): - assert match_capability_generalizations(model) is None, model + assert "mode" not in (match_capability_generalizations(model) or {}), model def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map): @@ -985,3 +1004,60 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None + + +@pytest.mark.parametrize( + "model,provider,sampling", + [ + ("gemini-3.6-flash-zeta", "gemini", False), + ("gemini-3.5-flash-lite-zeta", "vertex_ai", False), + ("gemini-3.10-pro-zeta", "gemini", False), + ("gemini-4-zeta", "gemini", False), + ("gemini-4.1-flash-zeta", "vertex_ai", False), + ("gemini-10-zeta", "gemini", False), + ("vertex_ai/gemini-4-zeta", None, False), + ("gemini-3.5-flash-zeta", "gemini", None), + ("gemini-3.5-pro-zeta", "vertex_ai", None), + ("gemini-3.1-pro-zeta", "gemini", None), + ("gemini-2.5-flash-zeta", "gemini", None), + ], +) +def test_shipped_gemini_sampling_rule_version_boundaries(shipped_cost_map, model, provider, sampling): + """The gemini-sampling-params-ignored rule marks Gemini 3.6 and every later 3.x minor, every 4+ major + and 3.5 Flash-Lite as not honoring temperature, top_p and top_k, and leaves 3.5 Flash, 3.5 Pro, 3.1 and + 2.5 ids without an opinion""" + assert model not in litellm.model_cost + info = litellm.get_model_info(model, custom_llm_provider=provider) + assert info.get("supports_sampling_params") is sampling, model + + +def test_shipped_gemini_sampling_rule_skips_non_text_ids(shipped_cost_map): + for model in ( + "gemini/gemini-4-flash-image", + "gemini/gemini-3.8-flash-tts", + "gemini/gemini-4-flash-live-preview", + "gemini/gemini-3.6-flash-native-audio", + "gemini/gemini-embedding-4", + "gemini/gemini-4-flash-transcribe", + "gemini/gemini-3.6-flash-live-translate-preview", + "gemini/gemini-4-computer-use-preview", + "gemini/gemini-zeta-9", + ): + assert (match_capability_generalizations(model) or {}).get("supports_sampling_params") is None, model + + +def test_shipped_gemini_sampling_rule_fills_mapped_google_rows_without_the_flag(shipped_cost_map): + """A mapped Gemini row past Google's cut reads as supports_sampling_params: false on AI Studio and Vertex + whether or not someone flagged it by hand, and a reseller copy of the same model is left alone""" + for key, model, provider in ( + ("gemini/gemini-4-argon", "gemini-4-argon", "gemini"), + ("vertex_ai/gemini-4-argon", "gemini-4-argon", "vertex_ai"), + ("gemini/gemini-3.6-flash", "gemini-3.6-flash", "gemini"), + ): + assert key in litellm.model_cost + assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_sampling_params"] is False, key + + assert match_fill_missing_generalizations("gemini-4-argon", "vertex_ai-language-models") == { + "supports_sampling_params": False + } + assert match_fill_missing_generalizations("gemini-4-argon", "github_copilot") is None diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index cf4a7c4cbc8..25ede9f0d01 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,25 +5,26 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping -from typing import Final +from collections.abc import Iterable, Mapping from types import MappingProxyType +from typing import Final +from unittest.mock import MagicMock from urllib.parse import parse_qs, urlparse import httpx import pytest +from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter -from unittest.mock import MagicMock import litellm +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, _openai_batch_jsonl_entry_to_vertex_rows, _sanitize_gcp_label_value, ) -from litellm.types.llms.openai import OpenAIFileObject, HttpxBinaryResponseContent -from openai.types.file_deleted import FileDeleted +from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject @pytest.fixture @@ -171,9 +172,7 @@ class TestBatchObjectNaming: def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" - object_name = config._get_gcs_object_name_from_batch_jsonl( - [{"body": {"model": "gemini/7768560373388541952"}}] - ) + object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini/7768560373388541952"}}]) assert object_name.startswith("litellm-vertex-files/endpoints/7768560373388541952/") assert "publishers" not in object_name @@ -865,7 +864,6 @@ class TestVertexBatchOutputTransformation: assert config._try_transform_vertex_batch_output_to_openai(binary) == binary - class TestTryTransformDoesNotMutateCallerLoggingObj: """Regression tests: _try_transform_vertex_batch_output_to_openai must not mutate the caller's logging_obj (model, start_time, optional_params).""" @@ -2045,3 +2043,110 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py monkeypatch.setattr(litellm, "drop_params", False) (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS} + + +def _stamp_batch_deployment_row(monkeypatch: pytest.MonkeyPatch, model: str) -> None: + monkeypatch.setitem(litellm.model_cost, model, {"mode": "batch", "litellm_provider": "vertex_ai"}) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_batch_row_for_a_gemini_4_batch_deployment_raises_a_400_naming_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The proxy stamps a batch deployment's `mode: batch` onto the backend model's map row, so a Gemini + model whose name is past Google's cut must still be filled by the shipped sampling rule and not pass as a + non-chat row""" + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"): + _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_batch_row_for_a_gemini_4_batch_deployment_drops_the_knobs_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stamp_batch_deployment_row(monkeypatch, "vertex_ai/gemini-4-zeta") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _wrap_entries([_chat_entry("vertex_ai/gemini-4-zeta", _SAMPLING_KNOBS)]) + assert _generation_config(row) == {"max_output_tokens": 5} + + +def _google_rows_flagged_only_on_vertex(monkeypatch: pytest.MonkeyPatch, model_name: str) -> None: + monkeypatch.setitem(litellm.model_cost, f"gemini/{model_name}", {"litellm_provider": "gemini", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model_name}", + {"litellm_provider": "vertex_ai", "mode": "chat", "supports_sampling_params": False}, + ) + + +def _upload_rows( + entries: Iterable[Mapping[str, object]], deployment_model: str | None, deployment_drop_params: bool | None = None +) -> list[object]: + """The Vertex rows a jsonl upload stores, built through the create-file transform the way the proxy's + `?model=` branch runs it, with the deployment's model and drop_params in litellm_params""" + jsonl: Final = "\n".join(json.dumps(dict(entry)) for entry in entries).encode() + litellm_params: Final[dict[str, object]] = { + **({"model": deployment_model} if deployment_model else {}), + **({"drop_params": deployment_drop_params} if deployment_drop_params is not None else {}), + } + request: Final = VertexAIFilesConfig().transform_create_file_request( + model="", + create_file_data={"file": ("batch.jsonl", jsonl, "application/jsonl"), "purpose": "batch"}, + optional_params={}, + litellm_params=litellm_params, + ) + assert isinstance(request, dict) + body: Final = b"".join(request["streaming_media_upload"]["body_stream"].iter_bytes()) + return [json.loads(line) for line in body.decode().splitlines()] + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_is_judged_by_its_vertex_row( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A row copied from an AI Studio call keeps its `gemini/` prefix but runs on Vertex, so the Vertex row's flag + decides. Looked up under that prefix on the Vertex route it found no row and fell open, so the knobs were + stored even though the model ignores them""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"temperature, top_p, top_k.*drop_params"): + _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + + +@pytest.mark.parametrize("deployment_model", ["vertex_ai/gemini-zeta-9", None], ids=["proxy_deployment", "sdk_upload"]) +def test_batch_row_keeping_a_gemini_prefix_drops_the_knobs_under_global_drop_params( + deployment_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _upload_rows([_chat_entry("gemini/gemini-zeta-9", _SAMPLING_KNOBS)], deployment_model) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + +def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_the_deployment_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A batch deployment with `drop_params: true` in its litellm_params gets the knobs dropped the way a + real-time call on that deployment does, instead of a 400 that only the global switch could avoid""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + (row,) = _upload_rows( + [_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9", deployment_drop_params=True + ) + assert _generation_config(TypeAdapter(Mapping[str, object]).validate_python(row)) == {"max_output_tokens": 5} + + +def test_batch_row_is_judged_by_the_deployment_model_it_runs_on_not_the_id_it_was_written_with( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The stored GCS path binds the batch job to the deployment's model, so a row written for an older id that + still takes the knobs is gated for the model that actually executes it""" + _google_rows_flagged_only_on_vertex(monkeypatch, "gemini-zeta-9") + monkeypatch.setitem( + litellm.model_cost, "vertex_ai/gemini-zeta-1", {"litellm_provider": "vertex_ai", "mode": "chat"} + ) + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(VertexAIError, match=r"gemini-zeta-9 does not support temperature, top_p, top_k"): + _upload_rows([_chat_entry("gemini-zeta-1", _SAMPLING_KNOBS)], "vertex_ai/gemini-zeta-9") diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3bc864cc047..20314e2997d 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -4116,12 +4116,13 @@ def _gemini_request_body( custom_llm_provider: Literal["vertex_ai", "gemini"], optional_params: dict[str, object], litellm_params: dict[str, object], + model: str = "gemini-zeta-9", ) -> RequestBody: from litellm.llms.vertex_ai.gemini.transformation import transform_request_body return transform_request_body( messages=[{"role": "user", "content": "hi"}], - model="gemini-zeta-9", + model=model, optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, @@ -4217,13 +4218,21 @@ def test_gemini_keeps_forwarding_sampling_params_unless_the_model_map_flags_them assert body["generationConfig"] == _SAMPLING_PARAMS -@pytest.mark.parametrize("model_key", ["gemini-zeta-9", "vertex_ai/gemini-zeta-9", "gemini/gemini-zeta-9"]) -def test_gemini_sampling_flag_is_read_from_whichever_provider_row_carries_it( - model_key: str, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("config_cls", "model_key"), + [ + ("vertex", "gemini-zeta-9"), + ("vertex", "vertex_ai/gemini-zeta-9"), + ("studio", "gemini-zeta-9"), + ("studio", "gemini/gemini-zeta-9"), + ], +) +def test_gemini_sampling_flag_is_read_from_the_row_of_the_route_serving_the_call( + config_cls: Literal["vertex", "studio"], model_key: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setitem(litellm.model_cost, model_key, {"supports_sampling_params": False}) - result: Final = VertexGeminiConfig().map_openai_params( + result: Final = _gemini_config(config_cls).map_openai_params( non_default_params={"temperature": 0.3}, optional_params={}, model="gemini-zeta-9", @@ -4268,6 +4277,76 @@ def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_un assert optional_params["max_output_tokens"] == 10 +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +@pytest.mark.parametrize( + ("model", "forwarded"), + [ + ("gemini-3.6-flash-zeta", False), + ("gemini-3.5-flash-lite-zeta", False), + ("gemini-3.7-pro-zeta", False), + ("gemini-4-zeta", False), + ("gemini-4.1-flash-zeta", False), + ("gemini-10-zeta", False), + ("gemini-3.5-flash-zeta", True), + ("gemini-3.5-pro-zeta", True), + ("gemini-3.1-pro-zeta", True), + ("gemini-2.5-flash-zeta", True), + ("gemini-zeta-9", True), + ], +) +def test_gemini_model_unknown_to_the_map_is_judged_by_the_shipped_sampling_rule( + custom_llm_provider: Literal["vertex_ai", "gemini"], model: str, forwarded: bool +) -> None: + """Google ignores the knobs on Gemini 3.6 Flash, 3.5 Flash-Lite and everything later, so a model newer + than the map gets the same drop-or-400 contract from the gemini-sampling-params-ignored fallback + generalization instead of silently forwarding (#47015 follow-up)""" + from litellm.utils import get_optional_params + + optional_params: Final = get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, max_tokens=10, drop_params=True, **_SAMPLING_PARAMS + ) + body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}, model=model) + + knobs: Final = _SAMPLING_PARAMS if forwarded else {} + assert body["generationConfig"] == {**knobs, "max_output_tokens": 10}, model + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("row", "forwarded"), + [ + ({"mode": "chat"}, False), + ({}, False), + ({"mode": "batch"}, False), + ({"mode": "audio_speech"}, True), + ({"mode": "chat", "supports_sampling_params": True}, True), + ], + ids=[ + "chat_row_without_flag", + "row_without_mode", + "batch_deployment_row_without_flag", + "non_chat_row_without_flag", + "explicit_true", + ], +) +def test_map_row_for_a_gemini_4_model_decides_before_the_shipped_sampling_rule( + row: dict[str, object], forwarded: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """An explicit flag always wins, a row silent on it or stamped `mode: batch` by a proxy batch deployment + is filled by the shipped rule, and a row for another mode keeps forwarding because Google's notice covers + the text models""" + from litellm.utils import get_optional_params + + monkeypatch.setitem(litellm.model_cost, "gemini-4-zeta", row) + + optional_params: Final = get_optional_params( + model="gemini-4-zeta", custom_llm_provider="gemini", temperature=0.3, drop_params=True + ) + + assert ("temperature" in optional_params) is forwarded + + def test_media_resolution_from_detail_parameter(): """Test that OpenAI's detail parameter is correctly mapped to media_resolution""" from litellm.llms.vertex_ai.gemini.transformation import ( @@ -7308,7 +7387,7 @@ def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(capl def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( caplog: pytest.LogCaptureFixture, ) -> None: - unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + unmapped: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(100)) mapped: Final = _mapped_gemini_3_models()[0] config: Final = VertexGeminiConfig() @@ -7351,7 +7430,7 @@ def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + priced: Final = tuple(f"gemini-3.5-flash-{uuid.uuid4().hex}" for _ in range(20)) for name in priced: monkeypatch.setitem( litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..d2360a05016 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -585,7 +585,7 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"]) +@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"]) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it."""