mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(gemini): handle regional prefixes and disabled thinking on gemini 3+
- Strip regional/multi-region prefixes (au., eu., us., global., us-central1., etc.) in VertexGeminiConfig._is_gemini_3_or_newer, _supports_minimal_thinking_level, and get_supported_openai_params. - Ensure _map_thinking_param sets thinkingLevel to minimal (for Flash models supporting minimal thinking) or low (for Pro / 3.7+ Flash) when thinking is disabled or budget_tokens == 0 on Gemini 3+ models.
This commit is contained in:
parent
dbaae521f9
commit
59914547d7
2 changed files with 223 additions and 27 deletions
|
|
@ -650,14 +650,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# Fallback: delegate to parent for unknown types
|
||||
return super().get_json_schema_from_pydantic_object(response_format)
|
||||
|
||||
@staticmethod
|
||||
def _strip_regional_prefix(model: str) -> str:
|
||||
return re.sub(
|
||||
r"(^|/)(?:[a-z0-9_-]+\.)+(?=gem(?:ini|ma)-)",
|
||||
r"\1",
|
||||
model,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_gemini_3_or_newer(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Gemini 3 or newer.
|
||||
"""
|
||||
model_name: Final = model.split("/")[-1].lower()
|
||||
normalized_model: Final = VertexGeminiConfig._strip_regional_prefix(model).lower()
|
||||
model_name: Final = normalized_model.split("/")[-1]
|
||||
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
|
||||
model.startswith("gemini/") and not model_name.startswith("gemini-")
|
||||
normalized_model.startswith("gemini/") and not model_name.startswith("gemini-")
|
||||
)
|
||||
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
|
||||
return False
|
||||
|
|
@ -765,7 +775,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if self._supports_penalty_parameters(model):
|
||||
supported_params.extend(["frequency_penalty", "presence_penalty"])
|
||||
|
||||
if supports_reasoning(model) or self._is_gemini_3_or_newer(model):
|
||||
if supports_reasoning(self._strip_regional_prefix(model)) or self._is_gemini_3_or_newer(model):
|
||||
supported_params.append("reasoning_effort")
|
||||
supported_params.append("thinking")
|
||||
|
||||
|
|
@ -1356,10 +1366,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
@staticmethod
|
||||
def _supports_minimal_thinking_level(model: str) -> bool:
|
||||
lowered: Final = model.lower()
|
||||
is_gemini3_or_newer_flash: Final = VertexGeminiConfig._is_gemini_3_or_newer(model) and "flash" in lowered
|
||||
normalized_model: Final = VertexGeminiConfig._strip_regional_prefix(model)
|
||||
lowered: Final = normalized_model.lower()
|
||||
is_gemini3_or_newer_flash: Final = (
|
||||
VertexGeminiConfig._is_gemini_3_or_newer(normalized_model) and "flash" in lowered
|
||||
)
|
||||
return is_gemini3_or_newer_flash and not is_explicitly_disabled_factory(
|
||||
model=model, custom_llm_provider=None, key="supports_minimal_reasoning_effort"
|
||||
model=normalized_model, custom_llm_provider=None, key="supports_minimal_reasoning_effort"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1461,18 +1474,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
|
||||
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
if thinking_enabled:
|
||||
if thinking_budget == 0:
|
||||
params["includeThoughts"] = False
|
||||
else:
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
# Follow provider defaults unless explicitly opted into legacy behavior.
|
||||
if litellm.enable_gemini_default_thinking_level_low is True:
|
||||
params["thinkingLevel"] = (
|
||||
"minimal" if VertexGeminiConfig._supports_minimal_thinking_level(model) else "low"
|
||||
)
|
||||
default_low_thinking_level: Final = (
|
||||
"minimal" if VertexGeminiConfig._supports_minimal_thinking_level(model) else "low"
|
||||
)
|
||||
if thinking_enabled and thinking_budget != 0:
|
||||
params["includeThoughts"] = sends_thoughts
|
||||
# Follow provider defaults unless explicitly opted into legacy behavior.
|
||||
if litellm.enable_gemini_default_thinking_level_low is True:
|
||||
params["thinkingLevel"] = default_low_thinking_level
|
||||
else:
|
||||
# Thinking disabled
|
||||
params["thinkingLevel"] = default_low_thinking_level
|
||||
params["includeThoughts"] = False
|
||||
else:
|
||||
# For older Gemini models, use thinkingBudget
|
||||
|
|
|
|||
|
|
@ -33,8 +33,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
|||
normalize_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata
|
||||
from litellm.types.llms.vertex_ai import GeminiFinishReason, GeminiThinkingConfig, RequestBody, UsageMetadata
|
||||
from litellm.types.utils import ChoiceLogprobs, ModelInfo, ModelResponseStream, Usage
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
|
@ -328,7 +329,9 @@ def test_chat_audio_drop_params_keeps_every_voice_config_field_gemini_accepts(sp
|
|||
|
||||
def test_chat_audio_reads_snake_case_wrapper_beside_null_camel_case_wrapper():
|
||||
mapped: Final = VertexGeminiConfig().map_openai_params(
|
||||
non_default_params={"audio": {"format": "pcm16", "speechConfig": None, "speech_config": {"voice_name": "Kore"}}},
|
||||
non_default_params={
|
||||
"audio": {"format": "pcm16", "speechConfig": None, "speech_config": {"voice_name": "Kore"}}
|
||||
},
|
||||
optional_params={},
|
||||
model="gemini-2.5-flash-preview-tts",
|
||||
drop_params=False,
|
||||
|
|
@ -3396,9 +3399,7 @@ async def test_vertex_ai_streaming_bad_request_is_not_wrapped():
|
|||
async def dispatch_failure_handlers(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
async def dispatch_attempt_failure_handlers(
|
||||
self, exception: Exception, traceback_exception: str
|
||||
) -> None:
|
||||
async def dispatch_attempt_failure_handlers(self, exception: Exception, traceback_exception: str) -> None:
|
||||
return None
|
||||
|
||||
def attempt_scoped_copy(self) -> "DummyLogging":
|
||||
|
|
@ -3528,9 +3529,7 @@ def test_grounding_annotations_preserve_all_web_sources(indices: list[int], expe
|
|||
]
|
||||
metadata: Final = {
|
||||
"groundingChunks": chunks,
|
||||
"groundingSupports": [
|
||||
{"segment": {"startIndex": 4, "endIndex": 11}, "groundingChunkIndices": indices}
|
||||
],
|
||||
"groundingSupports": [{"segment": {"startIndex": 4, "endIndex": 11}, "groundingChunkIndices": indices}],
|
||||
}
|
||||
annotations: Final = VertexGeminiConfig._convert_grounding_metadata_to_annotations([metadata], "The passage")
|
||||
expected: Final = [
|
||||
|
|
@ -8821,7 +8820,7 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
|
|||
)
|
||||
|
||||
assert result_disabled.get("includeThoughts") is False
|
||||
assert "thinkingLevel" not in result_disabled or result_disabled.get("thinkingLevel") is None
|
||||
assert result_disabled.get("thinkingLevel") == "low"
|
||||
|
||||
thinking_param_zero: AnthropicThinkingParam = {
|
||||
"type": "enabled",
|
||||
|
|
@ -8834,7 +8833,7 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults():
|
|||
)
|
||||
|
||||
assert result_zero["includeThoughts"] is False
|
||||
assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None
|
||||
assert result_zero["thinkingLevel"] == "minimal"
|
||||
|
||||
result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param(
|
||||
thinking_param=thinking_param,
|
||||
|
|
@ -10777,3 +10776,189 @@ def test_grounding_support_preserves_every_web_source(chunk_indices: list[int])
|
|||
}
|
||||
for index, start, end in expected_sources
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_is_gemini_3", "expected_supports_minimal"),
|
||||
[
|
||||
("gemini-3-flash", True, True),
|
||||
("gemini-3-flash-preview", True, True),
|
||||
("gemini-3.1-flash", True, True),
|
||||
("gemini-3.1-flash-lite-preview", True, True),
|
||||
("gemini-3.5-flash", True, True),
|
||||
("gemini-3.5-flash-preview", True, True),
|
||||
("gemini-3.5-flash-lite", True, True),
|
||||
("au.gemini-3.5-flash", True, True),
|
||||
("eu.gemini-3.5-flash", True, True),
|
||||
("us.gemini-3.5-flash", True, True),
|
||||
("ca.gemini-3.5-flash", True, True),
|
||||
("jp.gemini-3.5-flash", True, True),
|
||||
("uk.gemini-3.5-flash", True, True),
|
||||
("in.gemini-3.5-flash", True, True),
|
||||
("sg.gemini-3.5-flash", True, True),
|
||||
("kr.gemini-3.5-flash", True, True),
|
||||
("global.gemini-3.5-flash", True, True),
|
||||
("apac.gemini-3.5-flash", True, True),
|
||||
("us-central1.gemini-3.5-flash", True, True),
|
||||
("europe-west4.gemini-3.5-flash", True, True),
|
||||
("australia-southeast1.gemini-3.5-flash", True, True),
|
||||
("asia-northeast1.gemini-3.5-flash", True, True),
|
||||
("southamerica-east1.gemini-3.5-flash", True, True),
|
||||
("me-central1.gemini-3.5-flash", True, True),
|
||||
("vertex_ai/au.gemini-3.5-flash", True, True),
|
||||
("vertex_ai/eu.gemini-3.5-flash", True, True),
|
||||
("vertex_ai/us-central1.gemini-3.5-flash", True, True),
|
||||
("gemini/au.gemini-3.5-flash", True, True),
|
||||
("gemini/gemini-3.5-flash-preview", True, True),
|
||||
("gemini-3-pro", True, False),
|
||||
("gemini-3-pro-preview", True, False),
|
||||
("gemini-3.1-pro", True, False),
|
||||
("gemini-3.1-pro-preview", True, False),
|
||||
("gemini-3.5-pro", True, False),
|
||||
("au.gemini-3.1-pro", True, False),
|
||||
("eu.gemini-3.5-pro", True, False),
|
||||
("us-central1.gemini-3.5-pro", True, False),
|
||||
("vertex_ai/au.gemini-3.5-pro", True, False),
|
||||
("gemini-2.5-flash", False, False),
|
||||
("au.gemini-2.5-flash", False, False),
|
||||
("eu.gemini-2.5-pro", False, False),
|
||||
("us.gemini-2.5-flash", False, False),
|
||||
("us-central1.gemini-2.0-flash", False, False),
|
||||
("vertex_ai/au.gemini-2.5-flash", False, False),
|
||||
("gemini/eu.gemini-2.5-flash", False, False),
|
||||
("eu.gemma-3-27b-it", False, False),
|
||||
],
|
||||
)
|
||||
def test_gemini_3_helpers_with_regional_prefixes(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
expected_is_gemini_3: bool,
|
||||
expected_supports_minimal: bool,
|
||||
) -> None:
|
||||
assert VertexGeminiConfig._is_gemini_3_or_newer(model) is expected_is_gemini_3
|
||||
assert VertexGeminiConfig._supports_minimal_thinking_level(model) is expected_supports_minimal
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"gemini-3.5-flash",
|
||||
"au.gemini-3.5-flash",
|
||||
"eu.gemini-3.5-flash",
|
||||
"us.gemini-3.5-flash",
|
||||
"global.gemini-3.5-flash",
|
||||
"us-central1.gemini-3.5-flash",
|
||||
"europe-west4.gemini-3.5-flash",
|
||||
"australia-southeast1.gemini-3.5-flash",
|
||||
"vertex_ai/au.gemini-3.5-flash",
|
||||
"gemini/au.gemini-3.5-flash",
|
||||
"gemini-3.5-flash-preview",
|
||||
"gemini-3.1-flash",
|
||||
"gemini-3-flash",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("reasoning_effort", "expected"),
|
||||
[
|
||||
("none", {"thinkingLevel": "minimal", "includeThoughts": False}),
|
||||
("disable", {"thinkingLevel": "minimal", "includeThoughts": False}),
|
||||
("minimal", {"thinkingLevel": "minimal", "includeThoughts": True}),
|
||||
("low", {"thinkingLevel": "low", "includeThoughts": True}),
|
||||
("medium", {"thinkingLevel": "medium", "includeThoughts": True}),
|
||||
("high", {"thinkingLevel": "high", "includeThoughts": True}),
|
||||
],
|
||||
)
|
||||
def test_gemini_3_flash_reasoning_effort_mapping_all_regions(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
reasoning_effort: str,
|
||||
expected: GeminiThinkingConfig,
|
||||
) -> None:
|
||||
result: Final = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
|
||||
reasoning_effort=reasoning_effort,
|
||||
model=model,
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"gemini-3.1-pro",
|
||||
"gemini-3.1-pro-preview",
|
||||
"gemini-3.5-pro",
|
||||
"au.gemini-3.1-pro",
|
||||
"eu.gemini-3.5-pro",
|
||||
"us-central1.gemini-3.5-pro",
|
||||
"vertex_ai/au.gemini-3.7-flash",
|
||||
"us-central1.gemini-3.8-flash",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("reasoning_effort", "expected"),
|
||||
[
|
||||
("none", {"thinkingLevel": "low", "includeThoughts": False}),
|
||||
("disable", {"thinkingLevel": "low", "includeThoughts": False}),
|
||||
("minimal", {"thinkingLevel": "low", "includeThoughts": True}),
|
||||
("low", {"thinkingLevel": "low", "includeThoughts": True}),
|
||||
("medium", {"thinkingLevel": "medium", "includeThoughts": True}),
|
||||
("high", {"thinkingLevel": "high", "includeThoughts": True}),
|
||||
],
|
||||
)
|
||||
def test_gemini_3_pro_and_37_38_flash_reasoning_effort_mapping_all_regions(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
reasoning_effort: str,
|
||||
expected: GeminiThinkingConfig,
|
||||
) -> None:
|
||||
result: Final = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
|
||||
reasoning_effort=reasoning_effort,
|
||||
model=model,
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_level"),
|
||||
[
|
||||
("gemini-3.5-flash", "minimal"),
|
||||
("au.gemini-3.5-flash", "minimal"),
|
||||
("eu.gemini-3.5-flash", "minimal"),
|
||||
("us.gemini-3.5-flash", "minimal"),
|
||||
("us-central1.gemini-3.5-flash", "minimal"),
|
||||
("europe-west4.gemini-3.5-flash", "minimal"),
|
||||
("australia-southeast1.gemini-3.5-flash", "minimal"),
|
||||
("vertex_ai/au.gemini-3.5-flash", "minimal"),
|
||||
("gemini/eu.gemini-3.5-flash", "minimal"),
|
||||
("gemini-3.5-flash-preview", "minimal"),
|
||||
("gemini-3.1-flash", "minimal"),
|
||||
("gemini-3-flash", "minimal"),
|
||||
("gemini-3.1-pro", "low"),
|
||||
("gemini-3.1-pro-preview", "low"),
|
||||
("gemini-3.5-pro", "low"),
|
||||
("au.gemini-3.5-pro", "low"),
|
||||
("gemini-3.8-flash", "low"),
|
||||
("au.gemini-3.8-flash", "low"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_param",
|
||||
[
|
||||
{"type": "disabled"},
|
||||
{"type": "enabled", "budget_tokens": 0},
|
||||
],
|
||||
)
|
||||
def test_map_thinking_param_disabled_or_zero_budget_gemini_3(
|
||||
local_model_cost_map: None,
|
||||
model: str,
|
||||
expected_level: str,
|
||||
thinking_param: AnthropicThinkingParam,
|
||||
) -> None:
|
||||
result: Final = VertexGeminiConfig._map_thinking_param(
|
||||
thinking_param=thinking_param,
|
||||
model=model,
|
||||
)
|
||||
assert result == {
|
||||
"thinkingLevel": expected_level,
|
||||
"includeThoughts": False,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue