From 02f06e07820bff857349ce2731c828d04cde3b9f Mon Sep 17 00:00:00 2001 From: yuwk <1729065730@qq.com> Date: Fri, 11 Sep 2026 13:20:57 +0800 Subject: [PATCH 1/4] fix(bedrock): map OpenAI video_url parts to Converse video blocks `BedrockConverseMessagesProcessor` maps `image_url`, `file` and `document` content parts but had no branch for `video_url`, so an OpenAI-style video part was dropped without an error: only the text block survived into the request body, and the model answered about nothing while `usage.prompt_tokens` stayed at the text-only count. `BedrockImageProcessor` already picks the block type from the mime type (video/* -> VideoBlock, image/* -> ImageBlock), so routing `video_url` through the same processor yields the video block Converse expects. Both the sync and the async message-building paths are handled. Repro before/after (messages -> Converse content block kinds): video_url (mp4) ['text'] -> ['text', 'video'] image_url + mp4 ['text', 'video'] (unchanged) Fixes #40681 --- .../prompt_templates/factory.py | 32 ++++ .../chat/test_converse_transformation.py | 164 ++++++++++++++++++ 2 files changed, 196 insertions(+) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ece619e3883..67d78c0cb8f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4371,6 +4371,21 @@ class BedrockConverseMessagesProcessor: image_url=image_url, format=format ) _parts.append(_part) + elif element["type"] == "video_url": + # see the sync path: video shares the image + # processor, which keys the block type off the + # mime type. + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = await BedrockImageProcessor.process_image_async( + image_url=video_url, format=video_format + ) + _parts.append(_part) elif element["type"] == "file": _part = await BedrockConverseMessagesProcessor._async_process_file_message( message=cast(ChatCompletionFileObject, element) @@ -4744,6 +4759,23 @@ def _bedrock_converse_messages_pt( format=format, ) _parts.append(_part) + elif element["type"] == "video_url": + # OpenAI `video_url` parts must reach Converse as + # video blocks too. `process_image_sync` picks the + # block type from the mime type (video/* -> video, + # image/* -> image), so a video shares this path. + video_element = element["video_url"] + if isinstance(video_element, dict): + video_url = video_element["url"] + video_format = video_element.get("format") + else: + video_url = video_element + video_format = None + _part = BedrockImageProcessor.process_image_sync( + image_url=video_url, + format=video_format, + ) + _parts.append(_part) elif element["type"] == "file": _part = BedrockConverseMessagesProcessor._process_file_message( message=cast(ChatCompletionFileObject, element) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 2e9ea90f3b8..20d6e191e9e 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json import os @@ -6953,3 +6954,166 @@ def test_transform_response_honors_json_mode_kwarg_when_optional_params_lack_it( ) assert result.choices[0].message.tool_calls is None assert json.loads(result.choices[0].message.content) == {"city": "Paris", "population": 2100000} + + +def _video_clip_b64() -> str: + """A minimal fake mp4 payload - the Converse path only inspects the mime type.""" + return base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + b"\xab" * 32).decode() + + +def test_bedrock_converse_user_video_url_becomes_video_block(): + """ + An OpenAI `video_url` part used to be dropped on the Converse path: only + the text block reached Bedrock, so the model answered about nothing while + `usage.prompt_tokens` stayed at the text-only count. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + video_block = blocks[1]["video"] + assert video_block["format"] == "mp4" + assert video_block["source"]["bytes"] == clip_b64 + + +def test_bedrock_converse_user_video_url_str_form_becomes_video_block(): + """`video_url` may also be a bare data uri instead of a mapping.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": f"data:video/webm;base64,{clip_b64}", + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["video"] + assert blocks[0]["video"]["format"] == "webm" + assert blocks[0]["video"]["source"]["bytes"] == clip_b64 + + +@pytest.mark.asyncio +async def test_bedrock_converse_user_video_url_becomes_video_block_async(): + """The async (acompletion) message path must map video_url the same way.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + +def test_bedrock_converse_image_url_still_becomes_image_block(): + """The video_url branch must not hijack ordinary image parts.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + ) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{png_b64}"}, + }, + ], + } + ] + + translated = _bedrock_converse_messages_pt( + messages=messages, model="amazon.nova-pro-v1:0", llm_provider="bedrock" + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "image"] + assert blocks[1]["image"]["format"] == "png" + + +def test_bedrock_converse_transform_request_keeps_video_url(): + """End-to-end request build: the video block survives into the wire body.""" + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": {"url": f"data:video/mp4;base64,{clip_b64}"}, + }, + ], + } + ] + + body = AmazonConverseConfig().transform_request( + model="amazon.nova-pro-v1:0", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + blocks = body["messages"][0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["source"]["bytes"] == clip_b64 From b767b7cbf468ad7288a4b169e19013d51b582f0a Mon Sep 17 00:00:00 2001 From: yuwk <1729065730@qq.com> Date: Fri, 11 Sep 2026 13:34:24 +0800 Subject: [PATCH 2/4] fix(bedrock): infer video content type from url extension A remote video url served with a missing or generic content type (binary/octet-stream) could not be resolved to a Bedrock video format, so the request raised before it was built. The extension map only covered image and document formats Add the nine video extensions Converse accepts (mp4, mov, mkv, webm, flv, mpeg, mpg, wmv, 3gp) to that map, so a remote clip resolves to a video block the same way a data uri already does Verified without network by composing the real helpers: a .mp4 url with a binary/octet-stream header now yields video/mp4, Bedrock format mp4 and a video block --- .../prompt_templates/common_utils.py | 9 +++++ .../prompt_templates/factory.py | 7 ---- ...ore_utils_prompt_templates_common_utils.py | 33 +++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2485896184e..47211067c77 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1565,6 +1565,15 @@ def infer_content_type_from_url_and_content( "png": "image/png", "gif": "image/gif", "webp": "image/webp", + "mp4": "video/mp4", + "mov": "video/mov", + "mkv": "video/mkv", + "webm": "video/webm", + "flv": "video/flv", + "mpeg": "video/mpeg", + "mpg": "video/mpg", + "wmv": "video/wmv", + "3gp": "video/3gp", # Document formats "pdf": "application/pdf", "csv": "text/csv", diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 67d78c0cb8f..493f19faf5b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4372,9 +4372,6 @@ class BedrockConverseMessagesProcessor: ) _parts.append(_part) elif element["type"] == "video_url": - # see the sync path: video shares the image - # processor, which keys the block type off the - # mime type. video_element = element["video_url"] if isinstance(video_element, dict): video_url = video_element["url"] @@ -4760,10 +4757,6 @@ def _bedrock_converse_messages_pt( ) _parts.append(_part) elif element["type"] == "video_url": - # OpenAI `video_url` parts must reach Converse as - # video blocks too. `process_image_sync` picks the - # block type from the mime type (video/* -> video, - # image/* -> image), so a video shares this path. video_element = element["video_url"] if isinstance(video_element, dict): video_url = video_element["url"] diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b5890d1a5b0..b3b5b08e8da 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + infer_content_type_from_url_and_content, is_encrypted_reasoning_block, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, @@ -1813,3 +1814,35 @@ class TestEncryptedReasoningReplay: strip_encrypted_reasoning_from_messages(messages) assert messages == before + + +@pytest.mark.parametrize( + "url,expected", + [ + ("https://example.com/clip.mp4", "video/mp4"), + ("https://example.com/clip.webm?X-Amz-Signature=abc123", "video/webm"), + ("https://example.com/clip.mov", "video/mov"), + ("https://example.com/clip.mkv", "video/mkv"), + ("https://example.com/clip.3gp", "video/3gp"), + ], +) +def test_infer_video_content_type_from_url_extension(url, expected): + assert ( + infer_content_type_from_url_and_content( + url=url, + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type="binary/octet-stream", + ) + == expected + ) + + +def test_infer_video_content_type_from_url_extension_without_header(): + assert ( + infer_content_type_from_url_and_content( + url="https://example.com/clip.mp4", + content=b"\x00\x00\x00\x18ftypmp42", + current_content_type=None, + ) + == "video/mp4" + ) From a88e0f5f3dfdc7fb432b811885aef805cd2e7323 Mon Sep 17 00:00:00 2001 From: yuwk <1729065730@qq.com> Date: Fri, 11 Sep 2026 13:56:50 +0800 Subject: [PATCH 3/4] test(bedrock): cover every video extension added to content type inference The parametrized case list only exercised five of the nine video extensions, so codecov flagged the remaining map entries as uncovered Add flv, mpeg, mpg and wmv so every added entry has a case --- .../test_litellm_core_utils_prompt_templates_common_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b3b5b08e8da..c84bcbd6ae9 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1824,6 +1824,10 @@ class TestEncryptedReasoningReplay: ("https://example.com/clip.mov", "video/mov"), ("https://example.com/clip.mkv", "video/mkv"), ("https://example.com/clip.3gp", "video/3gp"), + ("https://example.com/clip.flv", "video/flv"), + ("https://example.com/clip.mpeg", "video/mpeg"), + ("https://example.com/clip.mpg", "video/mpg"), + ("https://example.com/clip.wmv", "video/wmv"), ], ) def test_infer_video_content_type_from_url_extension(url, expected): From ddb3b4450c4af2eb527585e96a876b3c1aaceb4c Mon Sep 17 00:00:00 2001 From: yuwk <1729065730@qq.com> Date: Fri, 11 Sep 2026 14:05:40 +0800 Subject: [PATCH 4/4] test(bedrock): cover the bare string video url on the async path No test exercised the bare string form of video_url through the async dispatcher, so those two lines stayed uncovered Add the async counterpart of the existing string form test --- .../chat/test_converse_transformation.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 20d6e191e9e..feb00765706 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -7059,6 +7059,39 @@ async def test_bedrock_converse_user_video_url_becomes_video_block_async(): assert blocks[1]["video"]["format"] == "mp4" +async def test_bedrock_converse_user_video_url_str_form_becomes_video_block_async(): + """A bare string video_url must survive the async (acompletion) path too.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + clip_b64 = _video_clip_b64() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video."}, + { + "type": "video_url", + "video_url": f"data:video/mp4;base64,{clip_b64}", + }, + ], + } + ] + + translated = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + + blocks = translated[0]["content"] + assert [next(iter(block)) for block in blocks] == ["text", "video"] + assert blocks[1]["video"]["format"] == "mp4" + + def test_bedrock_converse_image_url_still_becomes_image_block(): """The video_url branch must not hijack ordinary image parts.""" from litellm.litellm_core_utils.prompt_templates.factory import (