diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 71bb05ee155..b83dbbe0b17 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -47,6 +47,7 @@ from litellm.types.llms.openai import ( ChatCompletionImageObject, ChatCompletionTextObject, ChatCompletionUserMessage, + ChatCompletionVideoObject, ) from litellm.types.llms.vertex_ai import * from litellm.types.llms.vertex_ai import ( @@ -737,6 +738,31 @@ def gemini_convert_messages_with_history( if _message_content is not None and isinstance(_message_content, list): _parts: list[PartType] = [] for element_idx, element in enumerate(_message_content): + match element["type"]: + case "video_url": + video_element = cast( # cast-ok: the discriminator identifies video content + ChatCompletionVideoObject, element + ) + video_source = video_element["video_url"] + video_url: str + video_detail: str | None + match video_source: + case str(): + video_url = video_source + video_detail = None + case _: + video_url = video_source["url"] + video_detail = video_source.get("detail") + _parts.append( + _process_gemini_media( + image_url=video_url, + media_resolution_enum=_convert_detail_to_media_resolution_enum(video_detail), + model=model, + vertex_project=vertex_project, + vertex_credentials=vertex_credentials, + ) + ) + continue if element["type"] == "text" and "text" in element and len(element["text"]) > 0: element = cast(ChatCompletionTextObject, element) _part = PartType(text=element["text"]) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b97724319c7..beb9d0e4746 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -19,10 +19,62 @@ from litellm.llms.vertex_ai.gemini.transformation import ( gemini_convert_messages_with_history, transform_request_body, ) +from litellm.types.llms.openai import ChatCompletionTextObject, ChatCompletionVideoObject from litellm.types.llms.vertex_ai import BlobType, ContentType, PartType from litellm.types.utils import Message +@pytest.mark.parametrize( + ("content", "expected_parts"), + [ + pytest.param( + [{"type": "video_url", "video_url": {"url": "data:video/quicktime;base64,dmlkZW8="}}], + [{"inline_data": {"data": "dmlkZW8=", "mime_type": "video/quicktime"}}, {"text": " "}], + id="video-only QuickTime after a model turn", + ), + pytest.param( + [ + {"type": "text", "text": "Read the meter"}, + {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,dmlkZW8="}}, + ], + [{"text": "Read the meter"}, {"inline_data": {"data": "dmlkZW8=", "mime_type": "video/mp4"}}], + id="MP4 with text", + ), + pytest.param( + [{"type": "video_url", "video_url": "gs://test-bucket/video.mp4"}], + [{"file_data": {"file_uri": "gs://test-bucket/video.mp4", "mime_type": "video/mp4"}}, {"text": " "}], + id="video URL as a string", + ), + pytest.param( + [{"type": "video_url", "video_url": {"url": "gs://test-bucket/video.mp4", "detail": "high"}}], + [ + { + "file_data": {"file_uri": "gs://test-bucket/video.mp4", "mime_type": "video/mp4"}, + "media_resolution": {"level": "MEDIA_RESOLUTION_HIGH"}, + }, + {"text": " "}, + ], + id="video detail reaches Gemini", + ), + ], +) +def test_video_url_preserves_user_turn_and_media( + content: list[ChatCompletionTextObject | ChatCompletionVideoObject], expected_parts: list[PartType] +) -> None: + result: Final = gemini_convert_messages_with_history( + messages=[ + {"role": "assistant", "content": "Please send a video"}, + {"role": "user", "content": content}, + ], + model="gemini-3.7-flash", + ) + + assert result == [ + {"role": "model", "parts": [{"text": "Please send a video"}]}, + {"role": "user", "parts": expected_parts}, + ] + + def test_check_if_part_exists_in_parts(): parts = [ {"text": "Hello", "thought": True},