From f3e1307ed94ec23024513642d1bd84fb87285339 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:26:52 +0000 Subject: [PATCH] fix(vertex_ai): log Gemini 3 sampling deprecation once per model per worker (#40868) * fix(vertex_ai): log Gemini 3 sampling deprecation once per process Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vertex_ai): bound the Gemini 3 sampling warn-once cache and test it by behavior * fix(vertex_ai): log the Gemini 3 sampling deprecation once even when first calls race Pin the warned-model cap to its own constant so the shared LRU size knob cannot turn the dedupe off, and serialize the cached call so concurrent first requests for one model log it once * fix(vertex_ai): stop logging the Gemini 3 sampling deprecation for new models once 64 are remembered * fix(vertex_ai): give each mapped Gemini 3 model its own sampling notice and share one for unmapped names * fix(vertex_ai): key the Gemini 3 sampling notice on the loaded catalog so runtime-priced names share one notice --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/vertex_ai/gemini/transformation.py | 2 +- .../vertex_and_google_ai_studio_gemini.py | 51 +++--- .../vertex_ai/gemini/test_transformation.py | 16 ++ ...test_vertex_and_google_ai_studio_gemini.py | 156 +++++++++++++++++- 4 files changed, 203 insertions(+), 22 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 7b832b321da..b6517a90250 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -868,7 +868,7 @@ def gemini_convert_messages_with_history( """ has_text_in_content = check_text_in_content(user_content) if has_text_in_content is False: - verbose_logger.warning( + verbose_logger.debug( "No text in user content. Adding a blank text to user content, to ensure Gemini doesn't fail the request. Relevant Issue - https://github.com/BerriAI/litellm/issues/5515" ) user_content.append( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ec40cdfd11f..a65ae9da3e9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -3,10 +3,11 @@ ## Initial implementation - covers gemini + image gen calls import json import re +import threading import time from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy -from functools import partial +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -28,6 +29,7 @@ from litellm.constants import ( ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, @@ -196,6 +198,29 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr ) +_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_sampling_deprecation_lock: Final = threading.Lock() + + +@lru_cache(maxsize=None) +def _log_gemini_sampling_params_deprecated(model: str) -> None: + verbose_logger.warning( + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, + ) + + +def _sampling_deprecation_notice_key(model: str) -> str: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + is_cataloged: Final = any(name in catalog for name in (model, f"vertex_ai/{model}", f"gemini/{model}")) + return model if is_cataloged else _SAMPLING_DEPRECATION_UNMAPPED_MODELS + + +def _warn_gemini_sampling_params_deprecated(model: str) -> None: + with _sampling_deprecation_lock: + _log_gemini_sampling_params_deprecated(_sampling_deprecation_notice_key(model)) + + def _cache_creation_tokens(logging_obj: LoggingClass) -> int: recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_creation_input_tokens", 0) return recorded if isinstance(recorded, int) and recorded > 0 else 0 @@ -1266,7 +1291,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) - gemini_sampling_params_warned: bool = False for param, value in non_default_params.items(): if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): @@ -1275,28 +1299,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).", model, ) - if not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + _warn_gemini_sampling_params_deprecated(model) optional_params["temperature"] = value elif param == "top_p": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_p"] = value elif param == "top_k": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_k"] = value elif ( param == "stream" and value is True diff --git a/tests/unit/llms/vertex_ai/gemini/test_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_transformation.py index 2ea02b776ab..ac2d43f64a1 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_transformation.py @@ -1,12 +1,15 @@ import json +import logging import uuid +from typing import Final from unittest.mock import Mock import httpx import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.vertex_ai.gemini import transformation from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -469,3 +472,16 @@ async def test_vertex_ai_async_transform_inlines_only_the_urls_gemini_cannot_fet {"file_data": {"mime_type": "application/pdf", "file_uri": files_api_pdf}}, ] assert sorted(async_only_image_fetch.fetched) == sorted([plain_http_png, extensionless_https]) + + +def test_image_only_user_content_logs_the_blank_text_fixup_at_debug(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}]} + ] + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + contents: Final = transformation.gemini_convert_messages_with_history(messages) + + fixups: Final = tuple(r for r in caplog.records if "No text in user content" in r.getMessage()) + assert tuple(r.levelno for r in fixups) == (logging.DEBUG,) + assert contents[0]["parts"][-1] == {"text": " "} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 636d4034ccc..0796e125817 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,7 +1,9 @@ import asyncio, importlib, os, uuid import json +import logging import re -from collections.abc import Awaitable, Callable, Mapping +import threading +from collections.abc import Awaitable, Callable, Iterator, Mapping from copy import deepcopy from typing import Final, List, Literal, cast, get_args from unittest.mock import MagicMock, Mock, patch @@ -13,7 +15,9 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation +from litellm._logging import verbose_logger from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -21,6 +25,7 @@ from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, Vert from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, + _log_gemini_sampling_params_deprecated, ) from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage @@ -6690,6 +6695,155 @@ def test_supported_reasoning_efforts_still_map(model): assert "thinkingConfig" in result +@pytest.fixture +def fresh_sampling_deprecation_cache() -> Iterator[None]: + _log_gemini_sampling_params_deprecated.cache_clear() + yield + _log_gemini_sampling_params_deprecated.cache_clear() + + +def _mapped_gemini_3_models() -> tuple[str, ...]: + return tuple(sorted(name for name in GetModelCostMap.loaded_model_cost_map() if name.startswith("gemini-3"))) + + +def _deprecation_messages(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple(r.getMessage() for r in caplog.records if "DeprecationWarning" in r.getMessage()) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("top_p", 0.9), ("top_k", 5))) +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_per_model( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + first, second = _mapped_gemini_3_models()[:2] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = tuple( + config.map_openai_params( + non_default_params={param: value}, optional_params={}, model=model, drop_params=True + ) + for model in (first, first, first, second) + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert f"({first})" in deprecations[0] + assert f"({second})" in deprecations[1] + assert all(m[param] == value for m in mapped) + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(caplog: pytest.LogCaptureFixture) -> None: + model: Final = _mapped_gemini_3_models()[0] + + def map_sampling_params() -> None: + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + racing_call: Final = threading.Thread(target=map_sampling_params) + + def start_racing_call_while_first_warning_logs(record: logging.LogRecord) -> bool: + if "DeprecationWarning" in record.getMessage() and racing_call.ident is None: + racing_call.start() + racing_call.join(timeout=0.5) + return True + + verbose_logger.addFilter(start_racing_call_while_first_warning_logs) + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + map_sampling_params() + racing_call.join() + finally: + verbose_logger.removeFilter(start_racing_call_while_first_warning_logs) + + assert len(_deprecation_messages(caplog)) == 1 + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( + caplog: pytest.LogCaptureFixture, +) -> None: + unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + mapped: Final = _mapped_gemini_3_models()[0] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in (*unmapped, mapped) * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert all(name not in deprecations[0] for name in unmapped) + assert f"({mapped})" in deprecations[1] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ai( + caplog: pytest.LogCaptureFixture, +) -> None: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + vertex_only: Final = next( + name.removeprefix("vertex_ai/") + for name in catalog + if name.startswith("vertex_ai/gemini-3") + and name.removeprefix("vertex_ai/") not in catalog + and f"gemini/{name.removeprefix('vertex_ai/')}" not in catalog + ) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=vertex_only, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert f"({vertex_only})" in deprecations[0] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + for name in priced: + monkeypatch.setitem( + litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + f"gemini/{name}", + {"input_cost_per_token": 1e-7, "output_cost_per_token": 1e-7, "litellm_provider": "gemini"}, + ) + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in priced * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert all(name not in deprecations[0] for name in priced) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.5), ("top_p", 0.9), ("top_k", 5))) +def test_gemini_2_5_sampling_params_log_no_deprecation( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model=f"gemini-2.5-flash-{uuid.uuid4().hex}", + drop_params=True, + ) + + assert mapped[param] == value + assert not any("DeprecationWarning" in r.getMessage() for r in caplog.records) + + def _generate_content_body() -> dict: return { "candidates": [