fix(bedrock): correct modelInput format for Converse API batch models (#21656)

litellm/llms/bedrock/files/transformation.py

@@ -202,52 +202,84 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         return optional_params
 
 
+    # Providers whose InvokeModel body uses the Converse API format
+    # (messages + inferenceConfig + image blocks). Nova is the primary
+    # example; add others here as they adopt the same schema.
+    CONVERSE_INVOKE_PROVIDERS = ("nova",)
+
     def _map_openai_to_bedrock_params(
         self,
         openai_request_body: Dict[str, Any],
         provider: Optional[str] = None,
     ) -> Dict[str, Any]:
         """
-        Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic
+        Transform OpenAI request body to Bedrock-compatible modelInput
+        parameters using existing transformation logic.
+
+        Routes to the correct per-provider transformation so that the
+        resulting dict matches the InvokeModel body that Bedrock expects
+        for batch inference.
         """
         from litellm.types.utils import LlmProviders
+
         _model = openai_request_body.get("model", "")
         messages = openai_request_body.get("messages", [])
-        
-        # Use existing Anthropic transformation logic for Anthropic models
+        optional_params = {
+            k: v
+            for k, v in openai_request_body.items()
+            if k not in ["model", "messages"]
+        }
+
+        # --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
         if provider == LlmProviders.ANTHROPIC:
             from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
                 AmazonAnthropicClaudeConfig,
             )
-            
-            anthropic_config = AmazonAnthropicClaudeConfig()
-            
-            # Extract optional params (everything except model and messages)
-            optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
-            mapped_params = anthropic_config.map_openai_params(
+
+            config = AmazonAnthropicClaudeConfig()
+            mapped_params = config.map_openai_params(
                 non_default_params={},
                 optional_params=optional_params,
                 model=_model,
-                drop_params=False
+                drop_params=False,
             )
-            
-            # Transform using existing Anthropic logic
-            bedrock_params = anthropic_config.transform_request(
+            return config.transform_request(
                 model=_model,
                 messages=messages,
                 optional_params=mapped_params,
                 litellm_params={},
-                headers={}
+                headers={},
             )
 
-            return bedrock_params
-        else:
-            # For other providers, use basic mapping
-            bedrock_params = {
-                "messages": messages,
-                **{k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
-            }
-            return bedrock_params
+        # --- Converse API providers (e.g. Nova): use AmazonConverseConfig
+        #     to correctly convert image_url blocks to Bedrock image format
+        #     and wrap inference params inside inferenceConfig. ---
+        if provider in self.CONVERSE_INVOKE_PROVIDERS:
+            from litellm.llms.bedrock.chat.converse_transformation import (
+                AmazonConverseConfig,
+            )
+
+            config = AmazonConverseConfig()
+            mapped_params = config.map_openai_params(
+                non_default_params=optional_params,
+                optional_params={},
+                model=_model,
+                drop_params=False,
+            )
+            return config.transform_request(
+                model=_model,
+                messages=messages,
+                optional_params=mapped_params,
+                litellm_params={},
+                headers={},
+            )
+
+        # --- All other providers: passthrough (OpenAI-compatible models
+        #     like openai.gpt-oss-*, qwen, deepseek, etc.) ---
+        return {
+            "messages": messages,
+            **optional_params,
+        }
 
     def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
         self, openai_jsonl_content: List[Dict[str, Any]]

tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl

@@ -1,2 +1,2 @@
-{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}}
-{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}}
+{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}}
+{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}}

tests/test_litellm/llms/bedrock/files/test_bedrock_files_transformation.py

@@ -88,3 +88,223 @@ class TestBedrockFilesTransformation:
         
         print(f"\n=== Expected output written to: {expected_output_path} ===")
 
+    def test_nova_text_only_uses_converse_format(self):
+        """
+        Test that Nova models produce Converse API format in batch modelInput.
+
+        Verifies that:
+        - max_tokens is wrapped inside inferenceConfig.maxTokens
+        - messages use Converse content block format
+        - No raw OpenAI keys (max_tokens, temperature) at the top level
+        """
+        from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+
+        config = BedrockFilesConfig()
+
+        openai_jsonl_content = [
+            {
+                "custom_id": "nova-text-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "us.amazon.nova-pro-v1:0",
+                    "messages": [
+                        {"role": "user", "content": "What is the capital of France?"}
+                    ],
+                    "max_tokens": 50,
+                    "temperature": 0.7,
+                },
+            }
+        ]
+
+        result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            openai_jsonl_content
+        )
+
+        assert len(result) == 1
+        record = result[0]
+        assert record["recordId"] == "nova-text-1"
+
+        model_input = record["modelInput"]
+
+        # Must have inferenceConfig with maxTokens, NOT top-level max_tokens
+        assert "inferenceConfig" in model_input, (
+            "Nova modelInput must contain inferenceConfig"
+        )
+        assert model_input["inferenceConfig"]["maxTokens"] == 50
+        assert model_input["inferenceConfig"]["temperature"] == 0.7
+        assert "max_tokens" not in model_input, (
+            "max_tokens must NOT be at the top level for Nova"
+        )
+        assert "temperature" not in model_input, (
+            "temperature must NOT be at the top level for Nova"
+        )
+
+        # Must have messages
+        assert "messages" in model_input
+
+    def test_nova_image_content_uses_converse_image_blocks(self):
+        """
+        Test that image_url content blocks are converted to Bedrock Converse
+        image format for Nova models in batch.
+
+        Verifies that:
+        - image_url blocks are converted to {"image": {"format": ..., "source": {"bytes": ...}}}
+        - text blocks are converted to {"text": "..."}
+        - No raw OpenAI image_url type remains
+        """
+        from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+
+        config = BedrockFilesConfig()
+
+        # 1x1 transparent PNG
+        img_b64 = (
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4"
+            "2mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg=="
+        )
+
+        openai_jsonl_content = [
+            {
+                "custom_id": "nova-img-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "us.amazon.nova-pro-v1:0",
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "text", "text": "Describe this image."},
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": "data:image/png;base64," + img_b64
+                                    },
+                                },
+                            ],
+                        }
+                    ],
+                    "max_tokens": 100,
+                },
+            }
+        ]
+
+        result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            openai_jsonl_content
+        )
+
+        assert len(result) == 1
+        model_input = result[0]["modelInput"]
+
+        # Check inferenceConfig
+        assert "inferenceConfig" in model_input
+        assert model_input["inferenceConfig"]["maxTokens"] == 100
+        assert "max_tokens" not in model_input
+
+        # Check messages structure
+        messages = model_input["messages"]
+        assert len(messages) == 1
+        content_blocks = messages[0]["content"]
+
+        # Should have text block and image block in Converse format
+        has_text = False
+        has_image = False
+        for block in content_blocks:
+            if "text" in block:
+                has_text = True
+            if "image" in block:
+                has_image = True
+                # Verify Converse image format
+                assert "format" in block["image"], (
+                    "Image block must have format field"
+                )
+                assert "source" in block["image"], (
+                    "Image block must have source field"
+                )
+                assert "bytes" in block["image"]["source"], (
+                    "Image source must have bytes field"
+                )
+            # Must NOT have OpenAI-style image_url
+            assert "image_url" not in block, (
+                "image_url must not appear in Converse format"
+            )
+            assert block.get("type") != "image_url", (
+                "type=image_url must not appear in Converse format"
+            )
+
+        assert has_text, "Should have a text content block"
+        assert has_image, "Should have an image content block"
+
+    def test_anthropic_still_works_after_nova_fix(self):
+        """
+        Regression test: ensure Anthropic models are still correctly
+        transformed after the Converse API provider changes.
+        """
+        from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+
+        config = BedrockFilesConfig()
+
+        openai_jsonl_content = [
+            {
+                "custom_id": "claude-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+                    "messages": [
+                        {"role": "system", "content": "You are helpful."},
+                        {"role": "user", "content": "Hello!"},
+                    ],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            openai_jsonl_content
+        )
+
+        assert len(result) == 1
+        model_input = result[0]["modelInput"]
+
+        # Anthropic should have anthropic_version
+        assert "anthropic_version" in model_input
+        assert "messages" in model_input
+        assert "max_tokens" in model_input
+
+    def test_openai_passthrough_still_works(self):
+        """
+        Regression test: ensure OpenAI-compatible models (e.g. gpt-oss)
+        still use passthrough format.
+        """
+        from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+
+        config = BedrockFilesConfig()
+
+        openai_jsonl_content = [
+            {
+                "custom_id": "openai-1",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": "openai.gpt-oss-120b-1:0",
+                    "messages": [
+                        {"role": "user", "content": "Hello!"},
+                    ],
+                    "max_tokens": 10,
+                },
+            }
+        ]
+
+        result = config._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+            openai_jsonl_content
+        )
+
+        assert len(result) == 1
+        model_input = result[0]["modelInput"]
+
+        # OpenAI-compatible should use passthrough: max_tokens at top level
+        assert "messages" in model_input
+        assert "max_tokens" in model_input
+        assert model_input["max_tokens"] == 10
+