Merge pull request #17662 from Chesars/fix/gemini-2x-media-resolution-detail-parameter

fix(gemini): support detail parameter for image resolution on Gemini 2.x models
This commit is contained in:
Cesar Garcia 2026-03-04 17:51:51 -03:00 • committed by GitHub
commit 35b9ec3a86
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 428 additions and 56 deletions

View file

@ -1562,13 +1562,18 @@ LiteLLM Supports the following image types passed in `url`
## Media Resolution Control (Images & Videos)
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
LiteLLM supports OpenAI's `detail` parameter for specifying the image resolution when using Gemini models. The behavior differs between Gemini versions:
| Gemini Version | Resolution Control | Behavior |
|----------------|-------------------|----------|
| Gemini 3+ | Per-part | Each image/video can have its own `detail` setting |
| Gemini 2.x (2.0, 2.5) | Global | The highest `detail` from all images is applied globally via `mediaResolution` in `generationConfig` |
**Supported `detail` values:**
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `media_resolution: "medium"`
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
- `"low"` - Maps to `MEDIA_RESOLUTION_LOW` (280 tokens for images, 70 tokens per frame for videos)
- `"medium"` - Maps to `MEDIA_RESOLUTION_MEDIUM`
- `"high"` - Maps to `MEDIA_RESOLUTION_HIGH` (1120 tokens for images)
- `"ultra_high"` - Maps to `MEDIA_RESOLUTION_ULTRA_HIGH`
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
**Usage Examples:**
@ -1605,8 +1610,9 @@ messages = [
}
]
# Works with both Gemini 2.x and 3+
response = completion(
model="gemini/gemini-3-pro-preview",
model="gemini/gemini-2.5-flash", # or gemini-3-pro-preview
messages=messages,
)
```
@ -1647,7 +1653,9 @@ response = completion(
</Tabs>
:::info
**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
**Gemini 3+ Per-Part Resolution:** Each image or video can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This works with both `image_url` and `file` content types.
**Gemini 2.x Global Resolution:** When multiple images have different `detail` values, LiteLLM uses the highest resolution found and applies it globally via `mediaResolution` in `generationConfig` (e.g., if one image has `"low"` and another has `"high"`, all images will use `"high"`).
:::
## Video Metadata Control

View file

@ -77,6 +77,60 @@ def _convert_detail_to_media_resolution_enum(
return None
def _get_highest_media_resolution(
current: Optional[str], new_detail: Optional[str]
) -> Optional[str]:
"""
Compare two media resolution values and return the highest one.
Resolution hierarchy: ultra_high > high > medium > low > None
"""
resolution_priority = {"ultra_high": 4, "high": 3, "medium": 2, "low": 1}
current_priority = resolution_priority.get(current, 0) if current else 0
new_priority = resolution_priority.get(new_detail, 0) if new_detail else 0
if new_priority > current_priority:
return new_detail
return current
def _extract_max_media_resolution_from_messages(
messages: List[AllMessageValues],
) -> Optional[str]:
"""
Extract the highest media resolution (detail) from image content in messages.
This is used to set the global media_resolution in generation_config for
Gemini 2.x models which don't support per-part media resolution.
Args:
messages: List of messages in OpenAI format
Returns:
The highest detail level found ("high", "low", or None)
"""
max_resolution: Optional[str] = None
for msg in messages:
content = msg.get("content")
if isinstance(content, list):
for item in content:
if not isinstance(item, dict):
continue
detail: Optional[str] = None
if item.get("type") == "image_url":
image_url = item.get("image_url")
if isinstance(image_url, dict):
detail = image_url.get("detail")
elif item.get("type") == "file":
file_obj = item.get("file")
if isinstance(file_obj, dict):
detail = file_obj.get("detail")
if detail:
max_resolution = _get_highest_media_resolution(
max_resolution, detail
)
return max_resolution
def _apply_gemini_3_metadata(
part: PartType,
model: Optional[str],
@ -84,7 +138,7 @@ def _apply_gemini_3_metadata(
video_metadata: Optional[Dict[str, Any]],
) -> PartType:
"""
Apply the unique media_resolution and video_metadata parameters of Gemini 3+
Apply the unique media_resolution and video_metadata parameters of Gemini 3+
"""
if model is None:
return part
@ -541,7 +595,7 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
data_dict[k] = v
def _transform_request_body(
def _transform_request_body( # noqa: PLR0915
messages: List[AllMessageValues],
model: str,
optional_params: dict,
@ -615,6 +669,19 @@ def _transform_request_body(
generation_config: Optional[GenerationConfig] = GenerationConfig(
**filtered_params
)
# For Gemini 2.x models, add media_resolution to generation_config (global)
# Gemini 3+ supports per-part media_resolution, but 2.x only supports global
# Gemini 1.x does not support mediaResolution at all
if "gemini-2" in model:
max_media_resolution = _extract_max_media_resolution_from_messages(messages)
if max_media_resolution:
media_resolution_value = _convert_detail_to_media_resolution_enum(
max_media_resolution
)
if media_resolution_value and generation_config is not None:
generation_config["mediaResolution"] = media_resolution_value["level"]
data = RequestBody(contents=content)
if system_instructions is not None:
data["system_instruction"] = system_instructions

View file

@ -214,6 +214,7 @@ class GenerationConfig(TypedDict, total=False):
responseModalities: List[GeminiResponseModalities]
imageConfig: GeminiImageConfig
thinkingConfig: GeminiThinkingConfig
mediaResolution: str
speechConfig: SpeechConfig

View file

@ -5,6 +5,8 @@ from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history,
_transform_request_body,
check_if_part_exists_in_parts,
_get_highest_media_resolution,
_extract_max_media_resolution_from_messages,
)
from litellm.types.llms.vertex_ai import BlobType
from litellm.types.utils import Message
@ -547,12 +549,306 @@ def test_dummy_signature_with_function_call_mode():
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
# Tests for media_resolution (detail parameter) handling - Issue #17084
class TestMediaResolution:
"""Tests for media_resolution handling in Gemini 2.x models"""
def test_get_highest_media_resolution_high_wins(self):
"""Test that 'high' resolution takes precedence over 'low'"""
assert _get_highest_media_resolution("low", "high") == "high"
assert _get_highest_media_resolution("high", "low") == "high"
assert _get_highest_media_resolution(None, "high") == "high"
assert _get_highest_media_resolution("high", None) == "high"
def test_get_highest_media_resolution_low_over_none(self):
"""Test that 'low' resolution takes precedence over None"""
assert _get_highest_media_resolution(None, "low") == "low"
assert _get_highest_media_resolution("low", None) == "low"
def test_get_highest_media_resolution_same_values(self):
"""Test handling of same resolution values"""
assert _get_highest_media_resolution("high", "high") == "high"
assert _get_highest_media_resolution("low", "low") == "low"
assert _get_highest_media_resolution(None, None) is None
def test_get_highest_media_resolution_medium(self):
"""Test that 'medium' resolution is correctly ranked between 'low' and 'high'"""
assert _get_highest_media_resolution("low", "medium") == "medium"
assert _get_highest_media_resolution("medium", "low") == "medium"
assert _get_highest_media_resolution("medium", "high") == "high"
assert _get_highest_media_resolution("high", "medium") == "high"
assert _get_highest_media_resolution(None, "medium") == "medium"
assert _get_highest_media_resolution("medium", None) == "medium"
def test_get_highest_media_resolution_ultra_high(self):
"""Test that 'ultra_high' resolution takes precedence over all others"""
assert _get_highest_media_resolution("high", "ultra_high") == "ultra_high"
assert _get_highest_media_resolution("ultra_high", "high") == "ultra_high"
assert _get_highest_media_resolution("medium", "ultra_high") == "ultra_high"
assert _get_highest_media_resolution("low", "ultra_high") == "ultra_high"
assert _get_highest_media_resolution(None, "ultra_high") == "ultra_high"
assert _get_highest_media_resolution("ultra_high", None) == "ultra_high"
def test_extract_max_media_resolution_single_image_high(self):
"""Test extraction of media resolution from single image with detail=high"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,abc123", "detail": "high"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) == "high"
def test_extract_max_media_resolution_single_image_low(self):
"""Test extraction of media resolution from single image with detail=low"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) == "low"
def test_extract_max_media_resolution_no_detail(self):
"""Test extraction when no detail parameter is provided"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,abc123"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) is None
def test_extract_max_media_resolution_multiple_images_mixed(self):
"""Test that highest resolution is returned when multiple images have different details"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Compare these images"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,def456", "detail": "high"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) == "high"
def test_extract_max_media_resolution_text_only(self):
"""Test extraction from messages with no images"""
messages = [
{"role": "user", "content": "Hello, how are you?"},
{"role": "assistant", "content": "I'm doing well!"},
]
assert _extract_max_media_resolution_from_messages(messages) is None
def test_transform_request_body_gemini_2x_adds_media_resolution(self):
"""Test that media_resolution is added to generationConfig for Gemini 2.x models"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
},
],
}
]
result = _transform_request_body(
messages=messages,
model="gemini-2.5-flash",
optional_params={},
custom_llm_provider="gemini",
litellm_params={},
cached_content=None,
)
assert "generationConfig" in result
assert "mediaResolution" in result["generationConfig"]
assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_HIGH"
def test_transform_request_body_gemini_2x_low_resolution(self):
"""Test that low media_resolution is correctly added for Gemini 2.x"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "low"},
},
],
}
]
result = _transform_request_body(
messages=messages,
model="gemini-2.5-flash",
optional_params={},
custom_llm_provider="gemini",
litellm_params={},
cached_content=None,
)
assert "generationConfig" in result
assert "mediaResolution" in result["generationConfig"]
assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_LOW"
def test_transform_request_body_gemini_3_no_global_media_resolution(self):
"""Test that Gemini 3 models don't add media_resolution to generationConfig (they use per-part)"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
},
],
}
]
result = _transform_request_body(
messages=messages,
model="gemini-3-pro-preview",
optional_params={},
custom_llm_provider="gemini",
litellm_params={},
cached_content=None,
)
# Gemini 3 should NOT have mediaResolution in generationConfig
# (it's handled per-part in the content transformation)
if "generationConfig" in result:
assert "mediaResolution" not in result["generationConfig"]
def test_transform_request_body_no_detail_no_media_resolution(self):
"""Test that no mediaResolution is added when detail is not specified"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
},
],
}
]
result = _transform_request_body(
messages=messages,
model="gemini-2.5-flash",
optional_params={},
custom_llm_provider="gemini",
litellm_params={},
cached_content=None,
)
# When no detail is specified, mediaResolution should not be in generationConfig
if "generationConfig" in result:
assert "mediaResolution" not in result["generationConfig"]
def test_extract_max_media_resolution_file_type_with_detail(self):
"""Test that detail is extracted from file content type, not just image_url"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this file?"},
{
"type": "file",
"file": {"url": "data:image/png;base64,abc123", "detail": "high"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) == "high"
def test_extract_max_media_resolution_mixed_image_and_file(self):
"""Test that highest detail is returned across both image_url and file types"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Compare these"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
},
{
"type": "file",
"file": {"url": "data:image/png;base64,def456", "detail": "high"},
},
],
}
]
assert _extract_max_media_resolution_from_messages(messages) == "high"
def test_transform_request_body_gemini_1x_no_media_resolution(self):
"""Test that Gemini 1.x models don't get mediaResolution in generationConfig"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What is this?"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
},
],
}
]
result = _transform_request_body(
messages=messages,
model="gemini-1.5-pro",
optional_params={},
custom_llm_provider="gemini",
litellm_params={},
cached_content=None,
)
# Gemini 1.x should NOT have mediaResolution (not supported)
if "generationConfig" in result:
assert "mediaResolution" not in result["generationConfig"]
def test_convert_tool_response_with_base64_image():
"""Test tool response with base64 data URI image."""
# Create a small test image (1x1 red pixel PNG)
test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
image_data_uri = f"data:image/png;base64,{test_image_base64}"
# Create tool message with image
tool_message = {
"role": "tool",
@ -568,7 +864,7 @@ def test_convert_tool_response_with_base64_image():
}
]
}
# Mock last message with tool calls
last_message_with_tool_calls = {
"tool_calls": [
@ -581,16 +877,16 @@ def test_convert_tool_response_with_base64_image():
}
]
}
# Convert tool response (returns list when image is present)
result = convert_to_gemini_tool_call_result(
tool_message, last_message_with_tool_calls
)
# Verify results - should be a list with 2 parts (function_response + inline_data)
assert isinstance(result, list), f"Expected list when image present, got {type(result)}"
assert len(result) == 2, f"Expected 2 parts, got {len(result)}"
# Find function_response part and inline_data part
function_response_part = None
inline_data_part = None
@ -599,7 +895,7 @@ def test_convert_tool_response_with_base64_image():
function_response_part = part
elif "inline_data" in part:
inline_data_part = part
# Check function_response exists
assert function_response_part is not None, "Missing function_response part"
function_response = function_response_part["function_response"]
@ -608,7 +904,7 @@ def test_convert_tool_response_with_base64_image():
# Verify JSON response is parsed correctly
assert "url" in function_response["response"]
assert function_response["response"]["url"] == "https://example.com"
# Check inline_data exists
assert inline_data_part is not None, "Missing inline_data part"
inline_data: BlobType = inline_data_part["inline_data"]
@ -624,7 +920,7 @@ def test_convert_tool_response_with_url_image():
# Use a publicly accessible test image URL
test_image_url = "https://via.placeholder.com/1x1.png"
tool_message = {
"role": "tool",
"tool_call_id": "call_test456",
@ -639,7 +935,7 @@ def test_convert_tool_response_with_url_image():
}
]
}
last_message_with_tool_calls = {
"tool_calls": [
{
@ -651,25 +947,25 @@ def test_convert_tool_response_with_url_image():
}
]
}
try:
result = convert_to_gemini_tool_call_result(
tool_message, last_message_with_tool_calls
)
# Should be a list with 2 parts when image is present
assert isinstance(result, list), f"Expected list when image present, got {type(result)}"
assert len(result) == 2, f"Expected 2 parts, got {len(result)}"
# Find parts
function_response_part = next(p for p in result if "function_response" in p)
inline_data_part = next(p for p in result if "inline_data" in p)
# Check function_response exists
assert function_response_part is not None, "Missing function_response part"
function_response = function_response_part["function_response"]
assert function_response["name"] == "type_text_at"
# Check inline_data exists (URL should be downloaded and converted)
assert inline_data_part is not None, "Missing inline_data part"
inline_data: BlobType = inline_data_part["inline_data"]
@ -692,7 +988,7 @@ def test_convert_tool_response_text_only():
}
]
}
last_message_with_tool_calls = {
"tool_calls": [
{
@ -704,14 +1000,14 @@ def test_convert_tool_response_text_only():
}
]
}
result = convert_to_gemini_tool_call_result(
tool_message, last_message_with_tool_calls
)
# Should be a single part (no list) when no image
assert not isinstance(result, list), "Should return single part when no image"
# Check function_response exists
assert "function_response" in result
function_response = result["function_response"]
@ -719,7 +1015,7 @@ def test_convert_tool_response_text_only():
# Verify JSON response is parsed correctly
assert "status" in function_response["response"]
assert function_response["response"]["status"] == "completed"
# Check inline_data does NOT exist (no image provided)
assert "inline_data" not in result
@ -727,12 +1023,12 @@ def test_convert_tool_response_text_only():
def test_file_data_field_order():
"""
Test that file_data fields are in the correct order (mime_type before file_uri).
The Gemini API is sensitive to field order in the file_data object.
This test verifies that mime_type comes before file_uri in both:
1. Dictionary key order
2. JSON serialization
Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order.
"""
import json
@ -742,25 +1038,25 @@ def test_file_data_field_order():
# Test with HTTPS URL and explicit format (audio file)
file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123"
format = "audio/mpeg"
result = _process_gemini_media(image_url=file_url, format=format)
# Verify the result has file_data
assert "file_data" in result
file_data = result["file_data"]
# Verify both fields are present
assert "mime_type" in file_data
assert "file_uri" in file_data
assert file_data["mime_type"] == "audio/mpeg"
assert file_data["file_uri"] == file_url
# Verify field order by checking dictionary keys
# In Python 3.7+, dict maintains insertion order
file_data_keys = list(file_data.keys())
assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \
"mime_type must come before file_uri in the file_data dict"
# Also verify by serializing to JSON string
json_str = json.dumps(file_data)
mime_type_pos = json_str.find('"mime_type"')
@ -777,17 +1073,17 @@ def test_file_data_field_order_gcs_urls():
# Test with GCS URL
gcs_url = "gs://bucket/audio.mp3"
result = _process_gemini_media(image_url=gcs_url)
# Verify the result has file_data
assert "file_data" in result
file_data = result["file_data"]
# Verify both fields are present
assert "mime_type" in file_data
assert "file_uri" in file_data
# Verify field order
file_data_keys = list(file_data.keys())
assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \
@ -797,11 +1093,11 @@ def test_file_data_field_order_gcs_urls():
def test_extract_file_data_with_path_object():
"""
Test that filename is correctly extracted from Path objects for MIME type detection.
When uploading files using Path objects (e.g., Path("speech.mp3")), the filename
must be extracted to enable proper MIME type detection. Without this, files get
uploaded with 'application/octet-stream' instead of the correct MIME type.
Related issue: Files uploaded with wrong MIME type cause Gemini API to reject
requests where the specified format doesn't match the uploaded file's MIME type.
"""
@ -817,23 +1113,23 @@ def test_extract_file_data_with_path_object():
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp:
tmp.write(b"fake mp3 content")
tmp_path = tmp.name
try:
# Test with Path object
path_obj = Path(tmp_path)
extracted = extract_file_data(path_obj)
# Verify filename was extracted
assert extracted["filename"] is not None
assert extracted["filename"].endswith(".mp3")
# Verify MIME type was correctly detected
assert extracted["content_type"] == "audio/mpeg", \
f"Expected 'audio/mpeg' but got '{extracted['content_type']}'"
# Verify content was read
assert extracted["content"] == b"fake mp3 content"
finally:
# Clean up temporary file
os.unlink(tmp_path)
@ -852,22 +1148,22 @@ def test_extract_file_data_with_string_path():
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
tmp.write(b"fake wav content")
tmp_path = tmp.name
try:
# Test with string path
extracted = extract_file_data(tmp_path)
# Verify filename was extracted
assert extracted["filename"] is not None
assert extracted["filename"].endswith(".wav")
# Verify MIME type was correctly detected (can be audio/wav or audio/x-wav depending on system)
assert extracted["content_type"] in ["audio/wav", "audio/x-wav"], \
f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'"
# Verify content was read
assert extracted["content"] == b"fake wav content"
finally:
# Clean up temporary file
os.unlink(tmp_path)
@ -883,9 +1179,9 @@ def test_extract_file_data_with_tuple_format():
filename = "test_audio.mp3"
content = b"test audio content"
content_type = "audio/mpeg"
extracted = extract_file_data((filename, content, content_type))
# Verify all fields are correct
assert extracted["filename"] == filename
assert extracted["content"] == content
@ -905,15 +1201,15 @@ def test_extract_file_data_fallback_to_octet_stream():
with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp:
tmp.write(b"unknown content")
tmp_path = tmp.name
try:
# Test with unknown file type
extracted = extract_file_data(tmp_path)
# Verify filename was extracted
assert extracted["filename"] is not None
assert extracted["filename"].endswith(".xyz123")
# Verify MIME type falls back to octet-stream
assert extracted["content_type"] == "application/octet-stream", \
f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'"