Merge ddb3b4450c into 883b722fd2

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1565,6 +1565,15 @@ def infer_content_type_from_url_and_content(
         "png": "image/png",
         "gif": "image/gif",
         "webp": "image/webp",
+        "mp4": "video/mp4",
+        "mov": "video/mov",
+        "mkv": "video/mkv",
+        "webm": "video/webm",
+        "flv": "video/flv",
+        "mpeg": "video/mpeg",
+        "mpg": "video/mpg",
+        "wmv": "video/wmv",
+        "3gp": "video/3gp",
         # Document formats
         "pdf": "application/pdf",
         "csv": "text/csv",

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -4371,6 +4371,18 @@ class BedrockConverseMessagesProcessor:
                                     image_url=image_url, format=format
                                 )
                                 _parts.append(_part)
+                            elif element["type"] == "video_url":
+                                video_element = element["video_url"]
+                                if isinstance(video_element, dict):
+                                    video_url = video_element["url"]
+                                    video_format = video_element.get("format")
+                                else:
+                                    video_url = video_element
+                                    video_format = None
+                                _part = await BedrockImageProcessor.process_image_async(
+                                    image_url=video_url, format=video_format
+                                )
+                                _parts.append(_part)
                             elif element["type"] == "file":
                                 _part = await BedrockConverseMessagesProcessor._async_process_file_message(
                                     message=cast(ChatCompletionFileObject, element)
@@ -4744,6 +4756,19 @@ def _bedrock_converse_messages_pt(
                                 format=format,
                             )
                             _parts.append(_part)
+                        elif element["type"] == "video_url":
+                            video_element = element["video_url"]
+                            if isinstance(video_element, dict):
+                                video_url = video_element["url"]
+                                video_format = video_element.get("format")
+                            else:
+                                video_url = video_element
+                                video_format = None
+                            _part = BedrockImageProcessor.process_image_sync(
+                                image_url=video_url,
+                                format=video_format,
+                            )
+                            _parts.append(_part)
                         elif element["type"] == "file":
                             _part = BedrockConverseMessagesProcessor._process_file_message(
                                 message=cast(ChatCompletionFileObject, element)

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py

@@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_format_from_file_id,
     handle_any_messages_to_chat_completion_str_messages_conversion,
     hoist_images_from_tool_messages,
+    infer_content_type_from_url_and_content,
     is_encrypted_reasoning_block,
     responses_reasoning_items_from_thinking_blocks,
     split_concatenated_json_objects,
@@ -1813,3 +1814,39 @@ class TestEncryptedReasoningReplay:
         strip_encrypted_reasoning_from_messages(messages)
 
         assert messages == before
+
+
+@pytest.mark.parametrize(
+    "url,expected",
+    [
+        ("https://example.com/clip.mp4", "video/mp4"),
+        ("https://example.com/clip.webm?X-Amz-Signature=abc123", "video/webm"),
+        ("https://example.com/clip.mov", "video/mov"),
+        ("https://example.com/clip.mkv", "video/mkv"),
+        ("https://example.com/clip.3gp", "video/3gp"),
+        ("https://example.com/clip.flv", "video/flv"),
+        ("https://example.com/clip.mpeg", "video/mpeg"),
+        ("https://example.com/clip.mpg", "video/mpg"),
+        ("https://example.com/clip.wmv", "video/wmv"),
+    ],
+)
+def test_infer_video_content_type_from_url_extension(url, expected):
+    assert (
+        infer_content_type_from_url_and_content(
+            url=url,
+            content=b"\x00\x00\x00\x18ftypmp42",
+            current_content_type="binary/octet-stream",
+        )
+        == expected
+    )
+
+
+def test_infer_video_content_type_from_url_extension_without_header():
+    assert (
+        infer_content_type_from_url_and_content(
+            url="https://example.com/clip.mp4",
+            content=b"\x00\x00\x00\x18ftypmp42",
+            current_content_type=None,
+        )
+        == "video/mp4"
+    )

tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py

@@ -1,4 +1,5 @@
 import asyncio
+import base64
 import json
 import os
 
@@ -6953,3 +6954,199 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it(
     )
     assert result.choices[0].message.tool_calls is None
     assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000}
+
+
+def _video_clip_b64() -> str:
+    """A minimal fake mp4 payload - the Converse path only inspects the mime type."""
+    return base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + b"\xab" * 32).decode()
+
+
+def test_bedrock_converse_user_video_url_becomes_video_block():
+    """
+    An OpenAI `video_url` part used to be dropped on the Converse path: only
+    the text block reached Bedrock, so the model answered about nothing while
+    `usage.prompt_tokens` stayed at the text-only count.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    clip_b64 = _video_clip_b64()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this video."},
+                {
+                    "type": "video_url",
+                    "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"},
+                },
+            ],
+        }
+    ]
+
+    translated = _bedrock_converse_messages_pt(
+        messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock"
+    )
+
+    blocks = translated[0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["text", "video"]
+    video_block = blocks[1]["video"]
+    assert video_block["format"] == "mp4"
+    assert video_block["source"]["bytes"] == clip_b64
+
+
+def test_bedrock_converse_user_video_url_str_form_becomes_video_block():
+    """`video_url` may also be a bare data uri instead of a mapping."""
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    clip_b64 = _video_clip_b64()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "video_url",
+                    "video_url": f"data:video/webm;base64,{clip_b64}",
+                },
+            ],
+        }
+    ]
+
+    translated = _bedrock_converse_messages_pt(
+        messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock"
+    )
+
+    blocks = translated[0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["video"]
+    assert blocks[0]["video"]["format"] == "webm"
+    assert blocks[0]["video"]["source"]["bytes"] == clip_b64
+
+
+@pytest.mark.asyncio
+async def test_bedrock_converse_user_video_url_becomes_video_block_async():
+    """The async (acompletion) message path must map video_url the same way."""
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        BedrockConverseMessagesProcessor,
+    )
+
+    clip_b64 = _video_clip_b64()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this video."},
+                {
+                    "type": "video_url",
+                    "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"},
+                },
+            ],
+        }
+    ]
+
+    translated = (
+        await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
+            messages=messages,
+            model="amazon.nova-pro-v1:0",
+            llm_provider="bedrock",
+        )
+    )
+
+    blocks = translated[0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["text", "video"]
+    assert blocks[1]["video"]["format"] == "mp4"
+
+
+async def test_bedrock_converse_user_video_url_str_form_becomes_video_block_async():
+    """A bare string video_url must survive the async (acompletion) path too."""
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        BedrockConverseMessagesProcessor,
+    )
+
+    clip_b64 = _video_clip_b64()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this video."},
+                {
+                    "type": "video_url",
+                    "video_url": f"data:video/mp4;base64,{clip_b64}",
+                },
+            ],
+        }
+    ]
+
+    translated = (
+        await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
+            messages=messages,
+            model="amazon.nova-pro-v1:0",
+            llm_provider="bedrock",
+        )
+    )
+
+    blocks = translated[0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["text", "video"]
+    assert blocks[1]["video"]["format"] == "mp4"
+
+
+def test_bedrock_converse_image_url_still_becomes_image_block():
+    """The video_url branch must not hijack ordinary image parts."""
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    png_b64 = (
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
+    )
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this image."},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/png;base64,{png_b64}"},
+                },
+            ],
+        }
+    ]
+
+    translated = _bedrock_converse_messages_pt(
+        messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock"
+    )
+
+    blocks = translated[0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["text", "image"]
+    assert blocks[1]["image"]["format"] == "png"
+
+
+def test_bedrock_converse_transform_request_keeps_video_url():
+    """End-to-end request build: the video block survives into the wire body."""
+    clip_b64 = _video_clip_b64()
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Describe this video."},
+                {
+                    "type": "video_url",
+                    "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"},
+                },
+            ],
+        }
+    ]
+
+    body = AmazonConverseConfig().transform_request(
+        model="amazon.nova-pro-v1:0",
+        messages=messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+    )
+
+    blocks = body["messages"][0]["content"]
+    assert [next(iter(block)) for block in blocks] == ["text", "video"]
+    assert blocks[1]["video"]["source"]["bytes"] == clip_b64