From 10809a328727dbefbff5d0c6c66f34caceb3fcca Mon Sep 17 00:00:00 2001 From: Nguyen Khanh Nguyen Date: Thu, 15 Oct 2026 10:11:13 +0700 Subject: [PATCH 1/2] feat(gemini): forward media_processing on file parts --- .../llms/openai/chat/gpt_transformation.py | 2 +- .../llms/vertex_ai/gemini/transformation.py | 18 ++++++++--- litellm/types/llms/openai.py | 1 + litellm/types/llms/vertex_ai.py | 1 + .../test_vertex_ai_gemini_transformation.py | 31 +++++++++++++++++++ 5 files changed, 47 insertions(+), 6 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4d7ef833181..7d1d5cad2a3 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -307,7 +307,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): llm_provider="openai", ) new_file_obj: Final = ChatCompletionFileObjectFile( - **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} + **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} # pyright: ignore[reportArgumentType] # items() widens each value to object, but every key and value comes from the same ChatCompletionFileObjectFile ) content_item["file"] = new_file_obj diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 57327731863..9dfb5bec927 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -184,9 +184,10 @@ def _apply_gemini_metadata( model: str | None, media_resolution_enum: dict[str, str] | None, video_metadata: Mapping[str, object] | None, + media_processing: str | None = None, ) -> PartType: """ - Apply media_resolution and video_metadata parameters to a Gemini part. + Apply media_resolution, video_metadata and media_processing parameters to a Gemini part. - Per-part media_resolution: Gemini 3+ only (2.x uses generation_config global). - video_metadata (fps, startOffset, endOffset): all Gemini models (1.x, 2.x, 3+). @@ -206,6 +207,9 @@ def _apply_gemini_metadata( if gemini_video_metadata: part_dict["video_metadata"] = gemini_video_metadata + if media_processing is not None: + part_dict["media_processing"] = media_processing + return cast(PartType, part_dict) @@ -481,6 +485,7 @@ def _process_gemini_media( video_metadata: Mapping[str, object] | None = None, vertex_project: str | None = None, vertex_credentials: VERTEX_CREDENTIALS_TYPES | None = None, + media_processing: str | None = None, ) -> PartType: """ Given a media URL (image, audio, or video), return the appropriate PartType for Gemini @@ -492,6 +497,7 @@ def _process_gemini_media( media_resolution_enum: Media resolution level (for Gemini 3+) model: The model name (to check version compatibility) video_metadata: Video-specific metadata (fps, start_offset, end_offset) + media_processing: How Gemini processes the media, e.g. STATIC or AGENTIC """ try: @@ -556,7 +562,7 @@ def _process_gemini_media( ) file_data = FileDataType(mime_type=mime_type, file_uri=image_url) part: PartType = {"file_data": file_data} - return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata) + return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing) elif image_url.startswith(GEMINI_FILES_API_URI_PREFIX): # Gemini Files API URIs — the file is already uploaded to Google's # servers; pass the URI through as file_data without fetching it. @@ -568,16 +574,16 @@ def _process_gemini_media( # Gemini Files API references can be passed through as URI-only. file_data = cast(FileDataType, {"file_uri": image_url}) part = {"file_data": file_data} - return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata) + return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing) elif "https://" in image_url and (image_type := format or get_image_mime_type_from_url(image_url)) is not None: file_data = FileDataType(mime_type=image_type, file_uri=image_url) part = {"file_data": file_data} - return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata) + return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing) elif "http://" in image_url or "https://" in image_url or "base64" in image_url: image: Final = convert_to_anthropic_image_obj(image_url, format=format) _blob: Final[BlobType] = {"data": image["data"], "mime_type": image["media_type"]} part = {"inline_data": cast(BlobType, _blob)} - return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata) + return _apply_gemini_metadata(part, model, media_resolution_enum, video_metadata, media_processing) raise Exception(f"Invalid image received - {image_url}") except Exception as e: raise e @@ -821,6 +827,7 @@ def gemini_convert_messages_with_history( file_data = file_dict.get("file_data") detail = file_dict.get("detail") video_metadata = file_dict.get("video_metadata") + media_processing = file_dict.get("media_processing") passed_file = file_id or file_data if passed_file is None: raise Exception("Unknown file type. Please pass in a file_id or file_data") @@ -837,6 +844,7 @@ def gemini_convert_messages_with_history( video_metadata=video_metadata, vertex_project=vertex_project, vertex_credentials=vertex_credentials, + media_processing=media_processing, ) _parts.append(_part) except litellm.BadRequestError: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8731bc2905a..a8aa6a5452a 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -842,6 +842,7 @@ class ChatCompletionFileObjectFile(TypedDict, total=False): format: str detail: str # For video/image resolution control (low, medium, high, ultra_high) video_metadata: dict[str, Any] # For video-specific metadata (fps, start_offset, end_offset) + media_processing: ReadOnly[str] class ChatCompletionFileObject(TypedDict): diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 3dc7a56239d..84e4e3d1ca6 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -67,6 +67,7 @@ class PartType(TypedDict, total=False): thoughtSignature: str media_resolution: Literal["low", "medium", "high"] video_metadata: ReadOnly[VideoMetadataType] + media_processing: ReadOnly[str] class HttpxFunctionCall(TypedDict, total=False): diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b97724319c7..a8451fd1bec 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -19,6 +19,7 @@ from litellm.llms.vertex_ai.gemini.transformation import ( gemini_convert_messages_with_history, transform_request_body, ) +from litellm.types.llms.openai import ChatCompletionUserMessage from litellm.types.llms.vertex_ai import BlobType, ContentType, PartType from litellm.types.utils import Message @@ -1594,6 +1595,36 @@ class TestVideoMetadataAllGeminiModels: assert file_part["video_metadata"]["fps"] == 2 + +@pytest.mark.parametrize( + "file_id", + ["gs://bucket/video.mp4", "https://generativelanguage.googleapis.com/v1beta/files/abc123"], +) +def test_media_processing_is_forwarded_on_the_file_part(file_id: str): + message: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": [ + {"type": "text", "text": "Did the person try on a shoe?"}, + { + "type": "file", + "file": { + "file_id": file_id, + "format": "video/mp4", + "media_processing": "AGENTIC", + "video_metadata": {"start_offset": "5s"}, + }, + }, + ], + } + + contents: Final = gemini_convert_messages_with_history(messages=[message], model="gemini-3.5-flash") + + file_part: Final = next(part for part in contents[0]["parts"] if "file_data" in part) + assert "file_data" in file_part and file_part["file_data"]["file_uri"] == file_id + assert file_part.get("media_processing") == "AGENTIC" + assert file_part.get("video_metadata") == {"startOffset": "5s"} + + def test_convert_tool_response_with_base64_image(): """Test tool response with base64 data URI image.""" # Create a small test image (1x1 red pixel PNG) From 3b0e9a8784bbaff6dc5c75cb3e0fb93a72270c95 Mon Sep 17 00:00:00 2001 From: Nguyen Khanh Nguyen Date: Thu, 15 Oct 2026 10:11:13 +0700 Subject: [PATCH 2/2] chore: regenerate the OpenAPI snapshot and dashboard API types --- litellm/proxy/_lazy_openapi_snapshot.json | 4 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 ++ 2 files changed, 6 insertions(+) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 20819a59a43..31021370713 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -46533,6 +46533,10 @@ "title": "Format", "type": "string" }, + "media_processing": { + "title": "Media Processing", + "type": "string" + }, "video_metadata": { "additionalProperties": true, "title": "Video Metadata", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6f6eae1f98e..a1c179f5aae 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28699,6 +28699,8 @@ export interface components { filename?: string; /** Format */ format?: string; + /** Media Processing */ + media_processing?: string; /** Video Metadata */ video_metadata?: { [key: string]: unknown;