diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2485896184e..47211067c77 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1565,6 +1565,15 @@ def infer_content_type_from_url_and_content( "png": "image/png", "gif": "image/gif", "webp": "image/webp", + "mp4": "video/mp4", + "mov": "video/mov", + "mkv": "video/mkv", + "webm": "video/webm", + "flv": "video/flv", + "mpeg": "video/mpeg", + "mpg": "video/mpg", + "wmv": "video/wmv", + "3gp": "video/3gp", # Document formats "pdf": "application/pdf", "csv": "text/csv", diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ece619e3883..493f19faf5b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4371,6 +4371,18 @@ class BedrockConverseMessagesProcessor: image_url=image_url, format=format ) _parts.append(_part) + elif element["type"] == "video_url": + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = await BedrockImageProcessor.process_image_async( + image_url=video_url, format=video_format + ) + _parts.append(_part) elif element["type"] == "file": _part = await BedrockConverseMessagesProcessor._async_process_file_message( message=cast(ChatCompletionFileObject, element) @@ -4744,6 +4756,19 @@ def _bedrock_converse_messages_pt( format=format, ) _parts.append(_part) + elif element["type"] == "video_url": + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = BedrockImageProcessor.process_image_sync( + image_url=video_url, + format=video_format, + ) + _parts.append(_part) elif element["type"] == "file": _part = BedrockConverseMessagesProcessor._process_file_message( message=cast(ChatCompletionFileObject, element) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b5890d1a5b0..c84bcbd6ae9 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + infer_content_type_from_url_and_content, is_encrypted_reasoning_block, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, @@ -1813,3 +1814,39 @@ class TestEncryptedReasoningReplay: strip_encrypted_reasoning_from_messages(messages) assert messages == before + + +@pytest.mark.parametrize( + "url,expected", + [ + ("https://example.com/clip.mp4", "video/mp4"), + ("https://example.com/clip.webm?X-Amz-Signature=abc123", "video/webm"), + ("https://example.com/clip.mov", "video/mov"), + ("https://example.com/clip.mkv", "video/mkv"), + ("https://example.com/clip.3gp", "video/3gp"), + ("https://example.com/clip.flv", "video/flv"), + ("https://example.com/clip.mpeg", "video/mpeg"), + ("https://example.com/clip.mpg", "video/mpg"), + ("https://example.com/clip.wmv", "video/wmv"), + ], +) +def test_infer_video_content_type_from_url_extension(url, expected): + assert ( + infer_content_type_from_url_and_content( + url=url, + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type="binary/octet-stream", + ) + == expected + ) + + +def test_infer_video_content_type_from_url_extension_without_header(): + assert ( + infer_content_type_from_url_and_content( + url="https://example.com/clip.mp4", + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type=None, + ) + == "video/mp4" + ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 2e9ea90f3b8..feb00765706 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json import os @@ -6953,3 +6954,199 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it( ) assert result.choices[0].message.tool_calls is None assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000} + + +def _video_clip_b64() -> str: + """A minimal fake mp4 payload - the Converse path only inspects the mime type.""" + return base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + b"\xab" * 32).decode() + + +def test_bedrock_converse_user_video_url_becomes_video_block(): + """ + An OpenAI `video_url` part used to be dropped on the Converse path: only + the text block reached Bedrock, so the model answered about nothing while + `usage.prompt_tokens` stayed at the text-only count. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + video_block = blocks[1]["video"] + assert video_block["format"] == "mp4" + assert video_block["source"]["bytes"] == clip_b64 + + +def test_bedrock_converse_user_video_url_str_form_becomes_video_block(): + """`video_url` may also be a bare data uri instead of a mapping.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": f"data:video/webm;base64,{clip_b64}", + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["video"] + assert blocks[0]["video"]["format"] == "webm" + assert blocks[0]["video"]["source"]["bytes"] == clip_b64 + + +@pytest.mark.asyncio +async def test_bedrock_converse_user_video_url_becomes_video_block_async(): + """The async (acompletion) message path must map video_url the same way.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + +async def test_bedrock_converse_user_video_url_str_form_becomes_video_block_async(): + """A bare string video_url must survive the async (acompletion) path too.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": f"data:video/mp4;base64,{clip_b64}", + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + +def test_bedrock_converse_image_url_still_becomes_image_block(): + """The video_url branch must not hijack ordinary image parts.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + ) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{png_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "image"] + assert blocks[1]["image"]["format"] == "png" + + +def test_bedrock_converse_transform_request_keeps_video_url(): + """End-to-end request build: the video block survives into the wire body.""" + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + body = AmazonConverseConfig().transform_request( + model="amazon.nova-pro-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + blocks = body["messages"][0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["source"]["bytes"] == clip_b64