diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index e555d7e8ec0..8e5fff27f8a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1557,6 +1557,15 @@ def infer_content_type_from_url_and_content( "png": "image/png", "gif": "image/gif", "webp": "image/webp", + "mp4": "video/mp4", + "mov": "video/mov", + "mkv": "video/mkv", + "webm": "video/webm", + "flv": "video/flv", + "mpeg": "video/mpeg", + "mpg": "video/mpg", + "wmv": "video/wmv", + "3gp": "video/3gp", # Document formats "pdf": "application/pdf", "csv": "text/csv", diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c4e242fd360..f22a204ace5 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4402,6 +4402,18 @@ class BedrockConverseMessagesProcessor: image_url=image_url, format=format ) _parts.append(_part) + elif element["type"] == "video_url": + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = await BedrockImageProcessor.process_image_async( + image_url=video_url, format=video_format + ) + _parts.append(_part) elif element["type"] == "file": _part = await BedrockConverseMessagesProcessor._async_process_file_message( message=cast(ChatCompletionFileObject, element) @@ -4775,6 +4787,19 @@ def _bedrock_converse_messages_pt( format=format, ) _parts.append(_part) + elif element["type"] == "video_url": + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = BedrockImageProcessor.process_image_sync( + image_url=video_url, + format=video_format, + ) + _parts.append(_part) elif element["type"] == "file": _part = BedrockConverseMessagesProcessor._process_file_message( message=cast(ChatCompletionFileObject, element) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 45fc93f04c1..0732dc16c35 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + infer_content_type_from_url_and_content, is_encrypted_reasoning_block, merge_consecutive_system_messages, parse_tool_call_arguments, @@ -1898,6 +1899,40 @@ class TestEncryptedReasoningReplay: assert messages == before +@pytest.mark.parametrize( + "url,expected", + [ + ("https://example.com/clip.mp4", "video/mp4"), + ("https://example.com/clip.webm?X-Amz-Signature=abc123", "video/webm"), + ("https://example.com/clip.mov", "video/mov"), + ("https://example.com/clip.mkv", "video/mkv"), + ("https://example.com/clip.3gp", "video/3gp"), + ("https://example.com/clip.flv", "video/flv"), + ("https://example.com/clip.mpeg", "video/mpeg"), + ("https://example.com/clip.mpg", "video/mpg"), + ("https://example.com/clip.wmv", "video/wmv"), + ], +) +def test_infer_video_content_type_from_url_extension(url, expected): + assert ( + infer_content_type_from_url_and_content( + url=url, + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type="binary/octet-stream", + ) + == expected + ) + + +def test_infer_video_content_type_from_url_extension_without_header(): + assert ( + infer_content_type_from_url_and_content( + url="https://example.com/clip.mp4", + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type=None, + ) + == "video/mp4" + ) class TestMergeConsecutiveSystemMessages: def test_merges_each_run_of_string_system_messages_with_a_blank_line(self): messages = [ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 499096621c5..dad86404088 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,3 +1,5 @@ +import asyncio +import base64 import copy import json import os @@ -7478,6 +7480,200 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it( assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000} +def _video_clip_b64() -> str: + """A minimal fake mp4 payload - the Converse path only inspects the mime type.""" + return base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + b"\xab" * 32).decode() + + +def test_bedrock_converse_user_video_url_becomes_video_block(): + """ + An OpenAI `video_url` part used to be dropped on the Converse path: only + the text block reached Bedrock, so the model answered about nothing while + `usage.prompt_tokens` stayed at the text-only count. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + video_block = blocks[1]["video"] + assert video_block["format"] == "mp4" + assert video_block["source"]["bytes"] == clip_b64 + + +def test_bedrock_converse_user_video_url_str_form_becomes_video_block(): + """`video_url` may also be a bare data uri instead of a mapping.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": f"data:video/webm;base64,{clip_b64}", + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["video"] + assert blocks[0]["video"]["format"] == "webm" + assert blocks[0]["video"]["source"]["bytes"] == clip_b64 + + +@pytest.mark.asyncio +async def test_bedrock_converse_user_video_url_becomes_video_block_async(): + """The async (acompletion) message path must map video_url the same way.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + +async def test_bedrock_converse_user_video_url_str_form_becomes_video_block_async(): + """A bare string video_url must survive the async (acompletion) path too.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": f"data:video/mp4;base64,{clip_b64}", + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + +def test_bedrock_converse_image_url_still_becomes_image_block(): + """The video_url branch must not hijack ordinary image parts.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + ) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{png_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "image"] + assert blocks[1]["image"]["format"] == "png" + + +def test_bedrock_converse_transform_request_keeps_video_url(): + """End-to-end request build: the video block survives into the wire body.""" + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + body = AmazonConverseConfig().transform_request( + model="amazon.nova-pro-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + blocks = body["messages"][0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["source"]["bytes"] == clip_b64 FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14" EAGER_TOOL_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}