Merge 3b0e9a8784 into f768fb32ef

litellm/llms/openai/chat/gpt_transformation.py

@@ -307,7 +307,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
                     llm_provider="openai",
                 )
             new_file_obj: Final = ChatCompletionFileObjectFile(
-                **{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
+                **{k: v for k, v in file_obj.items() if k not in litellm_specific_params}  # pyright: ignore[reportArgumentType]  # items() widens each value to object, but every key and value comes from the same ChatCompletionFileObjectFile
             )
             content_item["file"] = new_file_obj
 

litellm/llms/vertex_ai/gemini/transformation.py

@@ -185,9 +185,10 @@ def _apply_gemini_metadata(
     model: str | None,
     media_resolution_enum: dict[str, str] | None,
     video_metadata: Mapping[str, object] | None,
+    media_processing: str | None = None,
 ) -> PartType:
     """
-    Apply media_resolution and video_metadata parameters to a Gemini part.
+    Apply media_resolution, video_metadata and media_processing parameters to a Gemini part.
 
     - Per-part media_resolution: Gemini 3+ only (2.x uses generation_config global).
     - video_metadata (fps, startOffset, endOffset): all Gemini models (1.x, 2.x, 3+).
@@ -207,6 +208,9 @@ def _apply_gemini_metadata(
         if gemini_video_metadata:
             part_dict["video_metadata"] = gemini_video_metadata
 
+    if media_processing is not None:
+        part_dict["media_processing"] = media_processing
+
     return cast(PartType, part_dict)
 
 
@@ -482,6 +486,7 @@ def _process_gemini_media(
     video_metadata: Mapping[str, object] | None = None,
     vertex_project: str | None = None,
     vertex_credentials: VERTEX_CREDENTIALS_TYPES | None = None,
+    media_processing: str | None = None,
 ) -> PartType:
     """
     Given a media URL (image, audio, or video), return the appropriate PartType for Gemini
@@ -493,6 +498,7 @@ def _process_gemini_media(
         media_resolution_enum: Media resolution level (for Gemini 3+)
         model: The model name (to check version compatibility)
         video_metadata: Video-specific metadata (fps, start_offset, end_offset)
+        media_processing: How Gemini processes the media, e.g. STATIC or AGENTIC
     """
 
     try:
@@ -557,7 +563,7 @@ def _process_gemini_media(
                 )
             file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
             part: PartType = {"file_data": file_data}
-            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata)
+            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing)
         elif image_url.startswith(GEMINI_FILES_API_URI_PREFIX):
             # Gemini Files API URIs — the file is already uploaded to Google's
             # servers; pass the URI through as file_data without fetching it.
@@ -569,16 +575,16 @@ def _process_gemini_media(
                 # Gemini Files API references can be passed through as URI-only.
                 file_data = cast(FileDataType, {"file_uri": image_url})
             part = {"file_data": file_data}
-            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata)
+            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing)
         elif "https://" in image_url and (image_type := format or get_image_mime_type_from_url(image_url)) is not None:
             file_data = FileDataType(mime_type=image_type, file_uri=image_url)
             part = {"file_data": file_data}
-            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata)
+            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing)
         elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
             image: Final = convert_to_anthropic_image_obj(image_url, format=format)
             _blob: Final[BlobType] = {"data": image["data"], "mime_type": image["media_type"]}
             part = {"inline_data": cast(BlobType, _blob)}
-            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata)
+            return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing)
         raise Exception(f"Invalid image received - {image_url}")
     except Exception as e:
         raise e
@@ -822,6 +828,7 @@ def gemini_convert_messages_with_history(
                             file_data = file_dict.get("file_data")
                             detail = file_dict.get("detail")
                             video_metadata = file_dict.get("video_metadata")
+                            media_processing = file_dict.get("media_processing")
                             passed_file = file_id or file_data
                             if passed_file is None:
                                 raise Exception("Unknown file type. Please pass in a file_id or file_data")
@@ -838,6 +845,7 @@ def gemini_convert_messages_with_history(
                                     video_metadata=video_metadata,
                                     vertex_project=vertex_project,
                                     vertex_credentials=vertex_credentials,
+                                    media_processing=media_processing,
                                 )
                                 _parts.append(_part)
                             except litellm.BadRequestError:

litellm/proxy/_lazy_openapi_snapshot.json

@@ -47702,6 +47702,10 @@
               "title": "Format",
               "type": "string"
             },
+            "media_processing": {
+              "title": "Media Processing",
+              "type": "string"
+            },
             "video_metadata": {
               "additionalProperties": true,
               "title": "Video Metadata",

litellm/types/llms/openai.py

@@ -849,6 +849,7 @@ class ChatCompletionFileObjectFile(TypedDict, total=False):
     format: str
     detail: str  # For video/image resolution control (low, medium, high, ultra_high)
     video_metadata: dict[str, Any]  # For video-specific metadata (fps, start_offset, end_offset)
+    media_processing: ReadOnly[str]
 
 
 class ChatCompletionFileObject(TypedDict):

litellm/types/llms/vertex_ai.py

@@ -68,6 +68,7 @@ class PartType(TypedDict, total=False):
     thoughtSignature: str
     media_resolution: Literal["low", "medium", "high"]
     video_metadata: ReadOnly[VideoMetadataType]
+    media_processing: ReadOnly[str]
 
 
 class HttpxFunctionCall(TypedDict, total=False):

tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py

@@ -19,6 +19,7 @@ from litellm.llms.vertex_ai.gemini.transformation import (
     gemini_convert_messages_with_history,
     transform_request_body,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.llms.vertex_ai import BlobType, ContentType, PartType
 from litellm.types.utils import Message
 
@@ -1594,6 +1595,36 @@ class TestVideoMetadataAllGeminiModels:
         assert file_part["video_metadata"]["fps"] == 2
 
 
+
+@pytest.mark.parametrize(
+    "file_id",
+    ["gs://bucket/video.mp4", "https://generativelanguage.googleapis.com/v1beta/files/abc123"],
+)
+def test_media_processing_is_forwarded_on_the_file_part(file_id: str):
+    message: Final[ChatCompletionUserMessage] = {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "Did the person try on a shoe?"},
+            {
+                "type": "file",
+                "file": {
+                    "file_id": file_id,
+                    "format": "video/mp4",
+                    "media_processing": "AGENTIC",
+                    "video_metadata": {"start_offset": "5s"},
+                },
+            },
+        ],
+    }
+
+    contents: Final = gemini_convert_messages_with_history(messages=[message], model="gemini-3.5-flash")
+
+    file_part: Final = next(part for part in contents[0]["parts"] if "file_data" in part)
+    assert "file_data" in file_part and file_part["file_data"]["file_uri"] == file_id
+    assert file_part.get("media_processing") == "AGENTIC"
+    assert file_part.get("video_metadata") == {"startOffset": "5s"}
+
+
 def test_convert_tool_response_with_base64_image():
     """Test tool response with base64 data URI image."""
     # Create a small test image (1x1 red pixel PNG)

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -28815,6 +28815,8 @@ export interface components {
             filename?: string;
             /** Format */
             format?: string;
+            /** Media Processing */
+            media_processing?: string;
             /** Video Metadata */
             video_metadata?: {
                 [key: string]: unknown;