From d3e24ab9cdf15883a9bb8e77a9faf1d994c62a42 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 7 Jan 2026 12:58:54 +0530 Subject: [PATCH] Fix: Gemini generate content request with audio file id --- .../prompt_templates/common_utils.py | 17 +- .../llms/vertex_ai/gemini/transformation.py | 2 +- .../test_vertex_ai_gemini_transformation.py | 186 ++++++++++++++++++ 3 files changed, 199 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b100b9b516b..2f8568db704 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,6 +6,7 @@ import io import mimetypes import re from os import PathLike +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -533,6 +534,12 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: # Convert content to bytes if isinstance(file_content, (str, PathLike)): # If it's a path, open and read the file + # Extract filename from path if not already set + if filename is None: + if isinstance(file_content, PathLike): + filename = Path(file_content).name + else: + filename = Path(str(file_content)).name with open(file_content, "rb") as f: content = f.read() elif isinstance(file_content, io.IOBase): @@ -550,11 +557,11 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: # Use provided content type or guess based on filename if not content_type: - content_type = ( - mimetypes.guess_type(filename)[0] - if filename - else "application/octet-stream" - ) + if filename: + guessed_type = mimetypes.guess_type(filename)[0] + content_type = guessed_type if guessed_type else "application/octet-stream" + else: + content_type = "application/octet-stream" return ExtractedFileData( filename=filename, diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 2bbdfa17cde..22042f7d641 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -115,7 +115,7 @@ def _process_gemini_image( and (image_type := format or _get_image_mime_type_from_url(image_url)) is not None ): - file_data = FileDataType(file_uri=image_url, mime_type=image_type) + file_data = FileDataType(mime_type=image_type, file_uri=image_url) part = {"file_data": file_data} if media_resolution_enum is not None and model is not None: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 3c3d68e8be8..70e6e9452e5 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -721,3 +721,189 @@ def test_convert_tool_response_text_only(): # Check inline_data does NOT exist (no image provided) assert "inline_data" not in result + + +def test_file_data_field_order(): + """ + Test that file_data fields are in the correct order (mime_type before file_uri). + + The Gemini API is sensitive to field order in the file_data object. + This test verifies that mime_type comes before file_uri in both: + 1. Dictionary key order + 2. JSON serialization + + Related issue: Gemini API returns 400 INVALID_ARGUMENT when fields are in wrong order. + """ + import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_image + + # Test with HTTPS URL and explicit format (audio file) + file_url = "https://generativelanguage.googleapis.com/v1beta/files/test123" + format = "audio/mpeg" + + result = _process_gemini_image(image_url=file_url, format=format) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + assert file_data["mime_type"] == "audio/mpeg" + assert file_data["file_uri"] == file_url + + # Verify field order by checking dictionary keys + # In Python 3.7+, dict maintains insertion order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \ + "mime_type must come before file_uri in the file_data dict" + + # Also verify by serializing to JSON string + json_str = json.dumps(file_data) + mime_type_pos = json_str.find('"mime_type"') + file_uri_pos = json_str.find('"file_uri"') + assert mime_type_pos < file_uri_pos, \ + "mime_type must appear before file_uri in JSON serialization" + + +def test_file_data_field_order_gcs_urls(): + """Test that GCS URLs also maintain correct field order.""" + import json + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_image + + # Test with GCS URL + gcs_url = "gs://bucket/audio.mp3" + + result = _process_gemini_image(image_url=gcs_url) + + # Verify the result has file_data + assert "file_data" in result + file_data = result["file_data"] + + # Verify both fields are present + assert "mime_type" in file_data + assert "file_uri" in file_data + + # Verify field order + file_data_keys = list(file_data.keys()) + assert file_data_keys.index("mime_type") < file_data_keys.index("file_uri"), \ + "mime_type must come before file_uri in the file_data dict" + + +def test_extract_file_data_with_path_object(): + """ + Test that filename is correctly extracted from Path objects for MIME type detection. + + When uploading files using Path objects (e.g., Path("speech.mp3")), the filename + must be extracted to enable proper MIME type detection. Without this, files get + uploaded with 'application/octet-stream' instead of the correct MIME type. + + Related issue: Files uploaded with wrong MIME type cause Gemini API to reject + requests where the specified format doesn't match the uploaded file's MIME type. + """ + from pathlib import Path + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary MP3 file + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as tmp: + tmp.write(b"fake mp3 content") + tmp_path = tmp.name + + try: + # Test with Path object + path_obj = Path(tmp_path) + extracted = extract_file_data(path_obj) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".mp3") + + # Verify MIME type was correctly detected + assert extracted["content_type"] == "audio/mpeg", \ + f"Expected 'audio/mpeg' but got '{extracted['content_type']}'" + + # Verify content was read + assert extracted["content"] == b"fake mp3 content" + + finally: + # Clean up temporary file + os.unlink(tmp_path) + + +def test_extract_file_data_with_string_path(): + """Test that filename is correctly extracted from string paths.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary WAV file + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(b"fake wav content") + tmp_path = tmp.name + + try: + # Test with string path + extracted = extract_file_data(tmp_path) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".wav") + + # Verify MIME type was correctly detected (can be audio/wav or audio/x-wav depending on system) + assert extracted["content_type"] in ["audio/wav", "audio/x-wav"], \ + f"Expected 'audio/wav' or 'audio/x-wav' but got '{extracted['content_type']}'" + + # Verify content was read + assert extracted["content"] == b"fake wav content" + + finally: + # Clean up temporary file + os.unlink(tmp_path) + + +def test_extract_file_data_with_tuple_format(): + """Test that tuple format (with explicit content_type) still works correctly.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + + # Test with tuple format: (filename, content, content_type) + filename = "test_audio.mp3" + content = b"test audio content" + content_type = "audio/mpeg" + + extracted = extract_file_data((filename, content, content_type)) + + # Verify all fields are correct + assert extracted["filename"] == filename + assert extracted["content"] == content + assert extracted["content_type"] == content_type + + +def test_extract_file_data_fallback_to_octet_stream(): + """Test that unknown file types fall back to application/octet-stream.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data + import tempfile + import os + + # Create a temporary file with unknown extension + with tempfile.NamedTemporaryFile(suffix=".xyz123", delete=False) as tmp: + tmp.write(b"unknown content") + tmp_path = tmp.name + + try: + # Test with unknown file type + extracted = extract_file_data(tmp_path) + + # Verify filename was extracted + assert extracted["filename"] is not None + assert extracted["filename"].endswith(".xyz123") + + # Verify MIME type falls back to octet-stream + assert extracted["content_type"] == "application/octet-stream", \ + f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" + + finally: + # Clean up temporary file + os.unlink(tmp_path)