mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Merge pull request #17662 from Chesars/fix/gemini-2x-media-resolution-detail-parameter
fix(gemini): support detail parameter for image resolution on Gemini 2.x models
This commit is contained in:
commit
35b9ec3a86
4 changed files with 428 additions and 56 deletions
|
|
@ -1562,13 +1562,18 @@ LiteLLM Supports the following image types passed in `url`
|
|||
|
||||
## Media Resolution Control (Images & Videos)
|
||||
|
||||
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
|
||||
LiteLLM supports OpenAI's `detail` parameter for specifying the image resolution when using Gemini models. The behavior differs between Gemini versions:
|
||||
|
||||
| Gemini Version | Resolution Control | Behavior |
|
||||
|----------------|-------------------|----------|
|
||||
| Gemini 3+ | Per-part | Each image/video can have its own `detail` setting |
|
||||
| Gemini 2.x (2.0, 2.5) | Global | The highest `detail` from all images is applied globally via `mediaResolution` in `generationConfig` |
|
||||
|
||||
**Supported `detail` values:**
|
||||
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
|
||||
- `"medium"` - Maps to `media_resolution: "medium"`
|
||||
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
|
||||
- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
|
||||
- `"low"` - Maps to `MEDIA_RESOLUTION_LOW` (280 tokens for images, 70 tokens per frame for videos)
|
||||
- `"medium"` - Maps to `MEDIA_RESOLUTION_MEDIUM`
|
||||
- `"high"` - Maps to `MEDIA_RESOLUTION_HIGH` (1120 tokens for images)
|
||||
- `"ultra_high"` - Maps to `MEDIA_RESOLUTION_ULTRA_HIGH`
|
||||
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
|
||||
|
||||
**Usage Examples:**
|
||||
|
|
@ -1605,8 +1610,9 @@ messages = [
|
|||
}
|
||||
]
|
||||
|
||||
# Works with both Gemini 2.x and 3+
|
||||
response = completion(
|
||||
model="gemini/gemini-3-pro-preview",
|
||||
model="gemini/gemini-2.5-flash", # or gemini-3-pro-preview
|
||||
messages=messages,
|
||||
)
|
||||
```
|
||||
|
|
@ -1647,7 +1653,9 @@ response = completion(
|
|||
</Tabs>
|
||||
|
||||
:::info
|
||||
**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
|
||||
**Gemini 3+ Per-Part Resolution:** Each image or video can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This works with both `image_url` and `file` content types.
|
||||
|
||||
**Gemini 2.x Global Resolution:** When multiple images have different `detail` values, LiteLLM uses the highest resolution found and applies it globally via `mediaResolution` in `generationConfig` (e.g., if one image has `"low"` and another has `"high"`, all images will use `"high"`).
|
||||
:::
|
||||
|
||||
## Video Metadata Control
|
||||
|
|
|
|||
|
|
@ -77,6 +77,60 @@ def _convert_detail_to_media_resolution_enum(
|
|||
return None
|
||||
|
||||
|
||||
def _get_highest_media_resolution(
|
||||
current: Optional[str], new_detail: Optional[str]
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Compare two media resolution values and return the highest one.
|
||||
Resolution hierarchy: ultra_high > high > medium > low > None
|
||||
"""
|
||||
resolution_priority = {"ultra_high": 4, "high": 3, "medium": 2, "low": 1}
|
||||
current_priority = resolution_priority.get(current, 0) if current else 0
|
||||
new_priority = resolution_priority.get(new_detail, 0) if new_detail else 0
|
||||
|
||||
if new_priority > current_priority:
|
||||
return new_detail
|
||||
return current
|
||||
|
||||
|
||||
def _extract_max_media_resolution_from_messages(
|
||||
messages: List[AllMessageValues],
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Extract the highest media resolution (detail) from image content in messages.
|
||||
|
||||
This is used to set the global media_resolution in generation_config for
|
||||
Gemini 2.x models which don't support per-part media resolution.
|
||||
|
||||
Args:
|
||||
messages: List of messages in OpenAI format
|
||||
|
||||
Returns:
|
||||
The highest detail level found ("high", "low", or None)
|
||||
"""
|
||||
max_resolution: Optional[str] = None
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
detail: Optional[str] = None
|
||||
if item.get("type") == "image_url":
|
||||
image_url = item.get("image_url")
|
||||
if isinstance(image_url, dict):
|
||||
detail = image_url.get("detail")
|
||||
elif item.get("type") == "file":
|
||||
file_obj = item.get("file")
|
||||
if isinstance(file_obj, dict):
|
||||
detail = file_obj.get("detail")
|
||||
if detail:
|
||||
max_resolution = _get_highest_media_resolution(
|
||||
max_resolution, detail
|
||||
)
|
||||
return max_resolution
|
||||
|
||||
|
||||
def _apply_gemini_3_metadata(
|
||||
part: PartType,
|
||||
model: Optional[str],
|
||||
|
|
@ -84,7 +138,7 @@ def _apply_gemini_3_metadata(
|
|||
video_metadata: Optional[Dict[str, Any]],
|
||||
) -> PartType:
|
||||
"""
|
||||
Apply the unique media_resolution and video_metadata parameters of Gemini 3+
|
||||
Apply the unique media_resolution and video_metadata parameters of Gemini 3+
|
||||
"""
|
||||
if model is None:
|
||||
return part
|
||||
|
|
@ -541,7 +595,7 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
|
|||
data_dict[k] = v
|
||||
|
||||
|
||||
def _transform_request_body(
|
||||
def _transform_request_body( # noqa: PLR0915
|
||||
messages: List[AllMessageValues],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
|
|
@ -615,6 +669,19 @@ def _transform_request_body(
|
|||
generation_config: Optional[GenerationConfig] = GenerationConfig(
|
||||
**filtered_params
|
||||
)
|
||||
|
||||
# For Gemini 2.x models, add media_resolution to generation_config (global)
|
||||
# Gemini 3+ supports per-part media_resolution, but 2.x only supports global
|
||||
# Gemini 1.x does not support mediaResolution at all
|
||||
if "gemini-2" in model:
|
||||
max_media_resolution = _extract_max_media_resolution_from_messages(messages)
|
||||
if max_media_resolution:
|
||||
media_resolution_value = _convert_detail_to_media_resolution_enum(
|
||||
max_media_resolution
|
||||
)
|
||||
if media_resolution_value and generation_config is not None:
|
||||
generation_config["mediaResolution"] = media_resolution_value["level"]
|
||||
|
||||
data = RequestBody(contents=content)
|
||||
if system_instructions is not None:
|
||||
data["system_instruction"] = system_instructions
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ class GenerationConfig(TypedDict, total=False):
|
|||
responseModalities: List[GeminiResponseModalities]
|
||||
imageConfig: GeminiImageConfig
|
||||
thinkingConfig: GeminiThinkingConfig
|
||||
mediaResolution: str
|
||||
speechConfig: SpeechConfig
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from litellm.llms.vertex_ai.gemini.transformation import (
|
|||
_gemini_convert_messages_with_history,
|
||||
_transform_request_body,
|
||||
check_if_part_exists_in_parts,
|
||||
_get_highest_media_resolution,
|
||||
_extract_max_media_resolution_from_messages,
|
||||
)
|
||||
from litellm.types.llms.vertex_ai import BlobType
|
||||
from litellm.types.utils import Message
|
||||
|
|
@ -547,12 +549,306 @@ def test_dummy_signature_with_function_call_mode():
|
|||
assert gemini_parts[0]["thoughtSignature"] == expected_dummy
|
||||
|
||||
|
||||
# Tests for media_resolution (detail parameter) handling - Issue #17084
|
||||
class TestMediaResolution:
|
||||
"""Tests for media_resolution handling in Gemini 2.x models"""
|
||||
|
||||
def test_get_highest_media_resolution_high_wins(self):
|
||||
"""Test that 'high' resolution takes precedence over 'low'"""
|
||||
assert _get_highest_media_resolution("low", "high") == "high"
|
||||
assert _get_highest_media_resolution("high", "low") == "high"
|
||||
assert _get_highest_media_resolution(None, "high") == "high"
|
||||
assert _get_highest_media_resolution("high", None) == "high"
|
||||
|
||||
def test_get_highest_media_resolution_low_over_none(self):
|
||||
"""Test that 'low' resolution takes precedence over None"""
|
||||
assert _get_highest_media_resolution(None, "low") == "low"
|
||||
assert _get_highest_media_resolution("low", None) == "low"
|
||||
|
||||
def test_get_highest_media_resolution_same_values(self):
|
||||
"""Test handling of same resolution values"""
|
||||
assert _get_highest_media_resolution("high", "high") == "high"
|
||||
assert _get_highest_media_resolution("low", "low") == "low"
|
||||
assert _get_highest_media_resolution(None, None) is None
|
||||
|
||||
def test_get_highest_media_resolution_medium(self):
|
||||
"""Test that 'medium' resolution is correctly ranked between 'low' and 'high'"""
|
||||
assert _get_highest_media_resolution("low", "medium") == "medium"
|
||||
assert _get_highest_media_resolution("medium", "low") == "medium"
|
||||
assert _get_highest_media_resolution("medium", "high") == "high"
|
||||
assert _get_highest_media_resolution("high", "medium") == "high"
|
||||
assert _get_highest_media_resolution(None, "medium") == "medium"
|
||||
assert _get_highest_media_resolution("medium", None) == "medium"
|
||||
|
||||
def test_get_highest_media_resolution_ultra_high(self):
|
||||
"""Test that 'ultra_high' resolution takes precedence over all others"""
|
||||
assert _get_highest_media_resolution("high", "ultra_high") == "ultra_high"
|
||||
assert _get_highest_media_resolution("ultra_high", "high") == "ultra_high"
|
||||
assert _get_highest_media_resolution("medium", "ultra_high") == "ultra_high"
|
||||
assert _get_highest_media_resolution("low", "ultra_high") == "ultra_high"
|
||||
assert _get_highest_media_resolution(None, "ultra_high") == "ultra_high"
|
||||
assert _get_highest_media_resolution("ultra_high", None) == "ultra_high"
|
||||
|
||||
def test_extract_max_media_resolution_single_image_high(self):
|
||||
"""Test extraction of media resolution from single image with detail=high"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc123", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) == "high"
|
||||
|
||||
def test_extract_max_media_resolution_single_image_low(self):
|
||||
"""Test extraction of media resolution from single image with detail=low"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) == "low"
|
||||
|
||||
def test_extract_max_media_resolution_no_detail(self):
|
||||
"""Test extraction when no detail parameter is provided"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc123"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) is None
|
||||
|
||||
def test_extract_max_media_resolution_multiple_images_mixed(self):
|
||||
"""Test that highest resolution is returned when multiple images have different details"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare these images"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,def456", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) == "high"
|
||||
|
||||
def test_extract_max_media_resolution_text_only(self):
|
||||
"""Test extraction from messages with no images"""
|
||||
messages = [
|
||||
{"role": "user", "content": "Hello, how are you?"},
|
||||
{"role": "assistant", "content": "I'm doing well!"},
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) is None
|
||||
|
||||
def test_transform_request_body_gemini_2x_adds_media_resolution(self):
|
||||
"""Test that media_resolution is added to generationConfig for Gemini 2.x models"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = _transform_request_body(
|
||||
messages=messages,
|
||||
model="gemini-2.5-flash",
|
||||
optional_params={},
|
||||
custom_llm_provider="gemini",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
)
|
||||
|
||||
assert "generationConfig" in result
|
||||
assert "mediaResolution" in result["generationConfig"]
|
||||
assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_HIGH"
|
||||
|
||||
def test_transform_request_body_gemini_2x_low_resolution(self):
|
||||
"""Test that low media_resolution is correctly added for Gemini 2.x"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "low"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = _transform_request_body(
|
||||
messages=messages,
|
||||
model="gemini-2.5-flash",
|
||||
optional_params={},
|
||||
custom_llm_provider="gemini",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
)
|
||||
|
||||
assert "generationConfig" in result
|
||||
assert "mediaResolution" in result["generationConfig"]
|
||||
assert result["generationConfig"]["mediaResolution"] == "MEDIA_RESOLUTION_LOW"
|
||||
|
||||
def test_transform_request_body_gemini_3_no_global_media_resolution(self):
|
||||
"""Test that Gemini 3 models don't add media_resolution to generationConfig (they use per-part)"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = _transform_request_body(
|
||||
messages=messages,
|
||||
model="gemini-3-pro-preview",
|
||||
optional_params={},
|
||||
custom_llm_provider="gemini",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
)
|
||||
|
||||
# Gemini 3 should NOT have mediaResolution in generationConfig
|
||||
# (it's handled per-part in the content transformation)
|
||||
if "generationConfig" in result:
|
||||
assert "mediaResolution" not in result["generationConfig"]
|
||||
|
||||
def test_transform_request_body_no_detail_no_media_resolution(self):
|
||||
"""Test that no mediaResolution is added when detail is not specified"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = _transform_request_body(
|
||||
messages=messages,
|
||||
model="gemini-2.5-flash",
|
||||
optional_params={},
|
||||
custom_llm_provider="gemini",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
)
|
||||
|
||||
# When no detail is specified, mediaResolution should not be in generationConfig
|
||||
if "generationConfig" in result:
|
||||
assert "mediaResolution" not in result["generationConfig"]
|
||||
|
||||
def test_extract_max_media_resolution_file_type_with_detail(self):
|
||||
"""Test that detail is extracted from file content type, not just image_url"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this file?"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"url": "data:image/png;base64,abc123", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) == "high"
|
||||
|
||||
def test_extract_max_media_resolution_mixed_image_and_file(self):
|
||||
"""Test that highest detail is returned across both image_url and file types"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare these"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,abc123", "detail": "low"},
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"url": "data:image/png;base64,def456", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert _extract_max_media_resolution_from_messages(messages) == "high"
|
||||
|
||||
def test_transform_request_body_gemini_1x_no_media_resolution(self):
|
||||
"""Test that Gemini 1.x models don't get mediaResolution in generationConfig"""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = _transform_request_body(
|
||||
messages=messages,
|
||||
model="gemini-1.5-pro",
|
||||
optional_params={},
|
||||
custom_llm_provider="gemini",
|
||||
litellm_params={},
|
||||
cached_content=None,
|
||||
)
|
||||
|
||||
# Gemini 1.x should NOT have mediaResolution (not supported)
|
||||
if "generationConfig" in result:
|
||||
assert "mediaResolution" not in result["generationConfig"]
|
||||
|
||||
|
||||
def test_convert_tool_response_with_base64_image():
|
||||
"""Test tool response with base64 data URI image."""
|
||||
# Create a small test image (1x1 red pixel PNG)
|
||||
test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
image_data_uri = f"data:image/png;base64,{test_image_base64}"
|
||||
|
||||
|
||||
# Create tool message with image
|
||||
tool_message = {
|
||||
"role": "tool",
|
||||
|
|
@ -568,7 +864,7 @@ def test_convert_tool_response_with_base64_image():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# Mock last message with tool calls
|
||||
last_message_with_tool_calls = {
|
||||
"tool_calls": [
|
||||
|
|
@ -581,16 +877,16 @@ def test_convert_tool_response_with_base64_image():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# Convert tool response (returns list when image is present)
|
||||
result = convert_to_gemini_tool_call_result(
|
||||
tool_message, last_message_with_tool_calls
|
||||
)
|
||||
|
||||
|
||||
# Verify results - should be a list with 2 parts (function_response + inline_data)
|
||||
assert isinstance(result, list), f"Expected list when image present, got {type(result)}"
|
||||
assert len(result) == 2, f"Expected 2 parts, got {len(result)}"
|
||||
|
||||
|
||||
# Find function_response part and inline_data part
|
||||
function_response_part = None
|
||||
inline_data_part = None
|
||||
|
|
@ -599,7 +895,7 @@ def test_convert_tool_response_with_base64_image():
|
|||
function_response_part = part
|
||||
elif "inline_data" in part:
|
||||
inline_data_part = part
|
||||
|
||||
|
||||
# Check function_response exists
|
||||
assert function_response_part is not None, "Missing function_response part"
|
||||
function_response = function_response_part["function_response"]
|
||||
|
|
@ -608,7 +904,7 @@ def test_convert_tool_response_with_base64_image():
|
|||
# Verify JSON response is parsed correctly
|
||||
assert "url" in function_response["response"]
|
||||
assert function_response["response"]["url"] == "https://example.com"
|
||||
|
||||
|
||||
# Check inline_data exists
|
||||
assert inline_data_part is not None, "Missing inline_data part"
|
||||
inline_data: BlobType = inline_data_part["inline_data"]
|
||||
|
|
@ -624,7 +920,7 @@ def test_convert_tool_response_with_url_image():
|
|||
|
||||
# Use a publicly accessible test image URL
|
||||
test_image_url = "https://via.placeholder.com/1x1.png"
|
||||
|
||||
|
||||
tool_message = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_test456",
|
||||
|
|
@ -639,7 +935,7 @@ def test_convert_tool_response_with_url_image():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
last_message_with_tool_calls = {
|
||||
"tool_calls": [
|
||||
{
|
||||
|
|
@ -651,25 +947,25 @@ def test_convert_tool_response_with_url_image():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
result = convert_to_gemini_tool_call_result(
|
||||
tool_message, last_message_with_tool_calls
|
||||
)
|
||||
|
||||
|
||||
# Should be a list with 2 parts when image is present
|
||||
assert isinstance(result, list), f"Expected list when image present, got {type(result)}"
|
||||
assert len(result) == 2, f"Expected 2 parts, got {len(result)}"
|
||||
|
||||
|
||||
# Find parts
|
||||
function_response_part = next(p for p in result if "function_response" in p)
|
||||
inline_data_part = next(p for p in result if "inline_data" in p)
|
||||
|
||||
|
||||
# Check function_response exists
|
||||
assert function_response_part is not None, "Missing function_response part"
|
||||
function_response = function_response_part["function_response"]
|
||||
assert function_response["name"] == "type_text_at"
|
||||
|
||||
|
||||
# Check inline_data exists (URL should be downloaded and converted)
|
||||
assert inline_data_part is not None, "Missing inline_data part"
|
||||
inline_data: BlobType = inline_data_part["inline_data"]
|
||||
|
|
@ -692,7 +988,7 @@ def test_convert_tool_response_text_only():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
last_message_with_tool_calls = {
|
||||
"tool_calls": [
|
||||
{
|
||||
|
|
@ -704,14 +1000,14 @@ def test_convert_tool_response_text_only():
|
|||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
result = convert_to_gemini_tool_call_result(
|
||||
tool_message, last_message_with_tool_calls
|
||||
)
|
||||
|
||||
|
||||
# Should be a single part (no list) when no image
|
||||
assert not isinstance(result, list), "Should return single part when no image"
|
||||
|
||||
|
||||
# Check function_response exists
|
||||
assert "function_response" in result
|
||||
function_response = result["function_response"]
|
||||
|
|
@ -719,7 +1015,7 @@ def test_convert_tool_response_text_only():
|
|||
# Verify JSON response is parsed correctly
|
||||
assert "status" in function_response["response"]
|
||||
assert function_response["response"]["status"] == "completed"
|
||||
|
||||
|
||||
# Check inline_data does NOT exist (no image provided)
|
||||
assert "inline_data" not in result
|
||||
|
||||
|
|
@ -727,12 +1023,12 @@ def test_convert_tool_response_text_only():
|
|||
def test_file_data_field_order():
|
||||
"""
|
||||
Test that file_data fields are in the correct order (mime_type before file_uri).
|
||||
|
||||
|
||||
The Gemini API is sensitive to field order in the file_data object.
|
||||
This test verifies that mime_type comes before file_uri in both:
|
||||
1. Dictionary key order
|
||||
2. JSON serialization
|
||||
|
||||
|
||||
Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order.
|
||||
"""
|
||||
import json
|
||||
|
|
@ -742,25 +1038,25 @@ def test_file_data_field_order():
|
|||
# Test with HTTPS URL and explicit format (audio file)
|
||||
file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123"
|
||||
format = "audio/mpeg"
|
||||
|
||||
|
||||
result = _process_gemini_media(image_url=file_url, format=format)
|
||||
|
||||
|
||||
# Verify the result has file_data
|
||||
assert "file_data" in result
|
||||
file_data = result["file_data"]
|
||||
|
||||
|
||||
# Verify both fields are present
|
||||
assert "mime_type" in file_data
|
||||
assert "file_uri" in file_data
|
||||
assert file_data["mime_type"] == "audio/mpeg"
|
||||
assert file_data["file_uri"] == file_url
|
||||
|
||||
|
||||
# Verify field order by checking dictionary keys
|
||||
# In Python 3.7+, dict maintains insertion order
|
||||
file_data_keys = list(file_data.keys())
|
||||
assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \
|
||||
"mime_type must come before file_uri in the file_data dict"
|
||||
|
||||
|
||||
# Also verify by serializing to JSON string
|
||||
json_str = json.dumps(file_data)
|
||||
mime_type_pos = json_str.find('"mime_type"')
|
||||
|
|
@ -777,17 +1073,17 @@ def test_file_data_field_order_gcs_urls():
|
|||
|
||||
# Test with GCS URL
|
||||
gcs_url = "gs://bucket/audio.mp3"
|
||||
|
||||
|
||||
result = _process_gemini_media(image_url=gcs_url)
|
||||
|
||||
|
||||
# Verify the result has file_data
|
||||
assert "file_data" in result
|
||||
file_data = result["file_data"]
|
||||
|
||||
|
||||
# Verify both fields are present
|
||||
assert "mime_type" in file_data
|
||||
assert "file_uri" in file_data
|
||||
|
||||
|
||||
# Verify field order
|
||||
file_data_keys = list(file_data.keys())
|
||||
assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \
|
||||
|
|
@ -797,11 +1093,11 @@ def test_file_data_field_order_gcs_urls():
|
|||
def test_extract_file_data_with_path_object():
|
||||
"""
|
||||
Test that filename is correctly extracted from Path objects for MIME type detection.
|
||||
|
||||
|
||||
When uploading files using Path objects (e.g., Path("speech.mp3")), the filename
|
||||
must be extracted to enable proper MIME type detection. Without this, files get
|
||||
uploaded with 'application/octet-stream' instead of the correct MIME type.
|
||||
|
||||
|
||||
Related issue: Files uploaded with wrong MIME type cause Gemini API to reject
|
||||
requests where the specified format doesn't match the uploaded file's MIME type.
|
||||
"""
|
||||
|
|
@ -817,23 +1113,23 @@ def test_extract_file_data_with_path_object():
|
|||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp:
|
||||
tmp.write(b"fake mp3 content")
|
||||
tmp_path = tmp.name
|
||||
|
||||
|
||||
try:
|
||||
# Test with Path object
|
||||
path_obj = Path(tmp_path)
|
||||
extracted = extract_file_data(path_obj)
|
||||
|
||||
|
||||
# Verify filename was extracted
|
||||
assert extracted["filename"] is not None
|
||||
assert extracted["filename"].endswith(".mp3")
|
||||
|
||||
|
||||
# Verify MIME type was correctly detected
|
||||
assert extracted["content_type"] == "audio/mpeg", \
|
||||
f"Expected 'audio/mpeg' but got '{extracted['content_type']}'"
|
||||
|
||||
|
||||
# Verify content was read
|
||||
assert extracted["content"] == b"fake mp3 content"
|
||||
|
||||
|
||||
finally:
|
||||
# Clean up temporary file
|
||||
os.unlink(tmp_path)
|
||||
|
|
@ -852,22 +1148,22 @@ def test_extract_file_data_with_string_path():
|
|||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
tmp.write(b"fake wav content")
|
||||
tmp_path = tmp.name
|
||||
|
||||
|
||||
try:
|
||||
# Test with string path
|
||||
extracted = extract_file_data(tmp_path)
|
||||
|
||||
|
||||
# Verify filename was extracted
|
||||
assert extracted["filename"] is not None
|
||||
assert extracted["filename"].endswith(".wav")
|
||||
|
||||
|
||||
# Verify MIME type was correctly detected (can be audio/wav or audio/x-wav depending on system)
|
||||
assert extracted["content_type"] in ["audio/wav", "audio/x-wav"], \
|
||||
f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'"
|
||||
|
||||
|
||||
# Verify content was read
|
||||
assert extracted["content"] == b"fake wav content"
|
||||
|
||||
|
||||
finally:
|
||||
# Clean up temporary file
|
||||
os.unlink(tmp_path)
|
||||
|
|
@ -883,9 +1179,9 @@ def test_extract_file_data_with_tuple_format():
|
|||
filename = "test_audio.mp3"
|
||||
content = b"test audio content"
|
||||
content_type = "audio/mpeg"
|
||||
|
||||
|
||||
extracted = extract_file_data((filename, content, content_type))
|
||||
|
||||
|
||||
# Verify all fields are correct
|
||||
assert extracted["filename"] == filename
|
||||
assert extracted["content"] == content
|
||||
|
|
@ -905,15 +1201,15 @@ def test_extract_file_data_fallback_to_octet_stream():
|
|||
with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp:
|
||||
tmp.write(b"unknown content")
|
||||
tmp_path = tmp.name
|
||||
|
||||
|
||||
try:
|
||||
# Test with unknown file type
|
||||
extracted = extract_file_data(tmp_path)
|
||||
|
||||
|
||||
# Verify filename was extracted
|
||||
assert extracted["filename"] is not None
|
||||
assert extracted["filename"].endswith(".xyz123")
|
||||
|
||||
|
||||
# Verify MIME type falls back to octet-stream
|
||||
assert extracted["content_type"] == "application/octet-stream", \
|
||||
f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue