diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 7b832b321da..b6517a90250 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -868,7 +868,7 @@ def gemini_convert_messages_with_history( """ has_text_in_content = check_text_in_content(user_content) if has_text_in_content is False: - verbose_logger.warning( + verbose_logger.debug( "No text in user content. Adding a blank text to user content, to ensure Gemini doesn't fail the request. Relevant Issue - https://github.com/BerriAI/litellm/issues/5515" ) user_content.append( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ec40cdfd11f..a65ae9da3e9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -3,10 +3,11 @@ ## Initial implementation - covers gemini + image gen calls import json import re +import threading import time from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy -from functools import partial +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -28,6 +29,7 @@ from litellm.constants import ( ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, @@ -196,6 +198,29 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr ) +_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_sampling_deprecation_lock: Final = threading.Lock() + + +@lru_cache(maxsize=None) +def _log_gemini_sampling_params_deprecated(model: str) -> None: + verbose_logger.warning( + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, + ) + + +def _sampling_deprecation_notice_key(model: str) -> str: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + is_cataloged: Final = any(name in catalog for name in (model, f"vertex_ai/{model}", f"gemini/{model}")) + return model if is_cataloged else _SAMPLING_DEPRECATION_UNMAPPED_MODELS + + +def _warn_gemini_sampling_params_deprecated(model: str) -> None: + with _sampling_deprecation_lock: + _log_gemini_sampling_params_deprecated(_sampling_deprecation_notice_key(model)) + + def _cache_creation_tokens(logging_obj: LoggingClass) -> int: recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_creation_input_tokens", 0) return recorded if isinstance(recorded, int) and recorded > 0 else 0 @@ -1266,7 +1291,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) - gemini_sampling_params_warned: bool = False for param, value in non_default_params.items(): if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): @@ -1275,28 +1299,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).", model, ) - if not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + _warn_gemini_sampling_params_deprecated(model) optional_params["temperature"] = value elif param == "top_p": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_p"] = value elif param == "top_k": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_k"] = value elif ( param == "stream" and value is True diff --git a/tests/unit/llms/vertex_ai/gemini/test_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_transformation.py index 2ea02b776ab..ac2d43f64a1 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_transformation.py @@ -1,12 +1,15 @@ import json +import logging import uuid +from typing import Final from unittest.mock import Mock import httpx import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.vertex_ai.gemini import transformation from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -469,3 +472,16 @@ async def test_vertex_ai_async_transform_inlines_only_the_urls_gemini_cannot_fet {"file_data": {"mime_type": "application/pdf", "file_uri": files_api_pdf}}, ] assert sorted(async_only_image_fetch.fetched) == sorted([plain_http_png, extensionless_https]) + + +def test_image_only_user_content_logs_the_blank_text_fixup_at_debug(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}]} + ] + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + contents: Final = transformation.gemini_convert_messages_with_history(messages) + + fixups: Final = tuple(r for r in caplog.records if "No text in user content" in r.getMessage()) + assert tuple(r.levelno for r in fixups) == (logging.DEBUG,) + assert contents[0]["parts"][-1] == {"text": " "} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 636d4034ccc..0796e125817 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,7 +1,9 @@ import asyncio, importlib, os, uuid import json +import logging import re -from collections.abc import Awaitable, Callable, Mapping +import threading +from collections.abc import Awaitable, Callable, Iterator, Mapping from copy import deepcopy from typing import Final, List, Literal, cast, get_args from unittest.mock import MagicMock, Mock, patch @@ -13,7 +15,9 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation +from litellm._logging import verbose_logger from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -21,6 +25,7 @@ from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, Vert from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, + _log_gemini_sampling_params_deprecated, ) from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage @@ -6690,6 +6695,155 @@ def test_supported_reasoning_efforts_still_map(model): assert "thinkingConfig" in result +@pytest.fixture +def fresh_sampling_deprecation_cache() -> Iterator[None]: + _log_gemini_sampling_params_deprecated.cache_clear() + yield + _log_gemini_sampling_params_deprecated.cache_clear() + + +def _mapped_gemini_3_models() -> tuple[str, ...]: + return tuple(sorted(name for name in GetModelCostMap.loaded_model_cost_map() if name.startswith("gemini-3"))) + + +def _deprecation_messages(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple(r.getMessage() for r in caplog.records if "DeprecationWarning" in r.getMessage()) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("top_p", 0.9), ("top_k", 5))) +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_per_model( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + first, second = _mapped_gemini_3_models()[:2] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = tuple( + config.map_openai_params( + non_default_params={param: value}, optional_params={}, model=model, drop_params=True + ) + for model in (first, first, first, second) + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert f"({first})" in deprecations[0] + assert f"({second})" in deprecations[1] + assert all(m[param] == value for m in mapped) + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(caplog: pytest.LogCaptureFixture) -> None: + model: Final = _mapped_gemini_3_models()[0] + + def map_sampling_params() -> None: + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + racing_call: Final = threading.Thread(target=map_sampling_params) + + def start_racing_call_while_first_warning_logs(record: logging.LogRecord) -> bool: + if "DeprecationWarning" in record.getMessage() and racing_call.ident is None: + racing_call.start() + racing_call.join(timeout=0.5) + return True + + verbose_logger.addFilter(start_racing_call_while_first_warning_logs) + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + map_sampling_params() + racing_call.join() + finally: + verbose_logger.removeFilter(start_racing_call_while_first_warning_logs) + + assert len(_deprecation_messages(caplog)) == 1 + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( + caplog: pytest.LogCaptureFixture, +) -> None: + unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + mapped: Final = _mapped_gemini_3_models()[0] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in (*unmapped, mapped) * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert all(name not in deprecations[0] for name in unmapped) + assert f"({mapped})" in deprecations[1] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ai( + caplog: pytest.LogCaptureFixture, +) -> None: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + vertex_only: Final = next( + name.removeprefix("vertex_ai/") + for name in catalog + if name.startswith("vertex_ai/gemini-3") + and name.removeprefix("vertex_ai/") not in catalog + and f"gemini/{name.removeprefix('vertex_ai/')}" not in catalog + ) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=vertex_only, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert f"({vertex_only})" in deprecations[0] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + for name in priced: + monkeypatch.setitem( + litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + f"gemini/{name}", + {"input_cost_per_token": 1e-7, "output_cost_per_token": 1e-7, "litellm_provider": "gemini"}, + ) + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in priced * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert all(name not in deprecations[0] for name in priced) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.5), ("top_p", 0.9), ("top_k", 5))) +def test_gemini_2_5_sampling_params_log_no_deprecation( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model=f"gemini-2.5-flash-{uuid.uuid4().hex}", + drop_params=True, + ) + + assert mapped[param] == value + assert not any("DeprecationWarning" in r.getMessage() for r in caplog.records) + + def _generate_content_body() -> dict: return { "candidates": [