Add support for output formatfor bedrock invoke via v1/messages

2026-10-07 02:59:05 +00:00 · 2026-01-22 16:36:03 +05:30 · 2026-01-22 16:36:03 +05:30 · 24faca9bcf
commit 24faca9bcf
parent b7b26492a8
3 changed files with 297 additions and 4 deletions
--- a/docs/my-website/docs/anthropic_unified/structured_output.md
+++ b/docs/my-website/docs/anthropic_unified/structured_output.md
@ -12,6 +12,7 @@ Use LiteLLM to call Anthropic's structured output feature via the `/v1/messages`
 | Anthropic | ✅ | Native support |
 | Azure AI (Anthropic models) | ✅ | Claude models on Azure AI |
 | Bedrock (Converse Anthropic models) | ✅ | Claude models via Bedrock Converse API |
+| Bedrock (Invoke Anthropic models) | ✅ | Claude models via Bedrock Invoke API |

 ## Usage

@ -133,7 +134,7 @@ curl http://localhost:4000/v1/messages \
 model_list:
  - model_name: bedrock-claude-sonnet
    litellm_params:
-      model: bedrock/anthropic.claude-sonnet-4-5-20250514-v1:0
+      model: bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0
      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
      aws_region_name: us-west-2
@ -178,6 +179,62 @@ curl http://localhost:4000/v1/messages \
  }'
 ```

+</TabItem>
+
+<TabItem value="bedrock_invoke" label="Bedrock (Invoke)">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: bedrock-claude-invoke
+    litellm_params:
+      model: bedrock/invoke/global.anthropic.claude-sonnet-4-5-20250929-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://localhost:4000/v1/messages \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H "anthropic-version: 2023-06-01" \
+  -d '{
+    "model": "bedrock-claude-invoke",
+    "max_tokens": 1024,
+    "messages": [
+      {
+        "role": "user",
+        "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
+      }
+    ],
+    "output_format": {
+      "type": "json_schema",
+      "schema": {
+        "type": "object",
+        "properties": {
+          "name": {"type": "string"},
+          "email": {"type": "string"},
+          "plan_interest": {"type": "string"},
+          "demo_requested": {"type": "boolean"}
+        },
+        "required": ["name", "email", "plan_interest", "demo_requested"],
+        "additionalProperties": false
+      }
+    }
+  }'
+```
+
+
 </TabItem>
 </Tabs>

--- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
+++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
@ -235,6 +235,63 @@ class AmazonAnthropicClaudeMessagesConfig(
            if "opus-4" in model.lower() or "opus_4" in model.lower():
                beta_set.add("tool-search-tool-2025-10-19")

+    def _convert_output_format_to_inline_schema(
+        self,
+        output_format: Dict,
+        anthropic_messages_request: Dict,
+    ) -> None:
+        """
+        Convert Anthropic output_format to inline schema in message content.
+        
+        Bedrock Invoke doesn't support the output_format parameter, so we embed
+        the schema directly into the user message content as text instructions.
+        
+        This approach adds the schema to the last user message, instructing the model
+        to respond in the specified JSON format.
+        
+        Args:
+            output_format: The output_format dict with 'type' and 'schema'
+            anthropic_messages_request: The request dict to modify in-place
+
+        Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/
+        """
+        import json
+        
+        # Extract schema from output_format
+        schema = output_format.get("schema")
+        if not schema:
+            return
+        
+        # Get messages from the request
+        messages = anthropic_messages_request.get("messages", [])
+        if not messages:
+            return
+        
+        # Find the last user message
+        last_user_message_idx = None
+        for idx in range(len(messages) - 1, -1, -1):
+            if messages[idx].get("role") == "user":
+                last_user_message_idx = idx
+                break
+        
+        if last_user_message_idx is None:
+            return
+        
+        last_user_message = messages[last_user_message_idx]
+        content = last_user_message.get("content", [])
+        
+        # Ensure content is a list
+        if isinstance(content, str):
+            content = [{"type": "text", "text": content}]
+            last_user_message["content"] = content
+        
+        # Add schema as text content to the message
+        schema_text = {
+            "type": "text",
+            "text": json.dumps(schema)
+        }
+        content.append(schema_text)
+
    def transform_anthropic_messages_request(
        self,
        model: str,
@ -272,9 +329,13 @@ class AmazonAnthropicClaudeMessagesConfig(
        # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it)
        self._remove_ttl_from_cache_control(anthropic_messages_request)

-        # 5. `output_format` is not supported on Bedrock invoke
-        if "output_format" in anthropic_messages_request:
-            anthropic_messages_request.pop("output_format", None)
+        # 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format)
+        output_format = anthropic_messages_request.pop("output_format", None)
+        if output_format:
+            self._convert_output_format_to_inline_schema(
+                output_format=output_format,
+                anthropic_messages_request=anthropic_messages_request,
+            )
            
        # 6. AUTO-INJECT beta headers based on features used
        anthropic_model_info = AnthropicModelInfo()
--- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
+++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
@ -104,3 +104,178 @@ def test_aws_params_filtered_from_request_body():
    # Verify messages are present
    assert "messages" in result, "messages should be in request body"
    assert len(result["messages"]) == 1, "should have 1 message"
+
+
+def test_output_format_conversion_to_inline_schema():
+    """
+    Test that output_format is converted to inline schema in message content for Bedrock Invoke.
+    
+    Bedrock Invoke doesn't support the output_format parameter, so LiteLLM converts it by
+    embedding the schema directly into the user message content.
+    """
+    from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
+        AmazonAnthropicClaudeMessagesConfig,
+    )
+    
+    config = AmazonAnthropicClaudeMessagesConfig()
+    
+    # Test messages
+    messages = [
+        {"role": "user", "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan."}
+    ]
+    
+    # Output format with schema
+    output_format_schema = {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "email": {"type": "string"},
+            "plan_interest": {"type": "string"}
+        },
+        "required": ["name", "email", "plan_interest"],
+        "additionalProperties": False
+    }
+    
+    anthropic_messages_optional_request_params = {
+        "max_tokens": 1024,
+        "output_format": {
+            "type": "json_schema",
+            "schema": output_format_schema
+        }
+    }
+    
+    # Transform the request
+    result = config.transform_anthropic_messages_request(
+        model="anthropic.claude-sonnet-4-20250514-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+        litellm_params={},
+        headers={},
+    )
+    
+    # Verify output_format was removed from the request
+    assert "output_format" not in result, "output_format should be removed from request body"
+    
+    # Verify the schema was added to the last user message content
+    assert "messages" in result
+    last_user_message = result["messages"][0]
+    assert last_user_message["role"] == "user"
+    
+    content = last_user_message["content"]
+    assert isinstance(content, list), "content should be a list"
+    assert len(content) == 2, "content should have 2 items (original text + schema)"
+    
+    # Check original text is preserved
+    assert content[0]["type"] == "text"
+    assert "John Smith" in content[0]["text"]
+    
+    # Check schema was added as JSON string
+    assert content[1]["type"] == "text"
+    schema_text = content[1]["text"]
+    
+    # Parse the schema JSON
+    parsed_schema = json.loads(schema_text)
+    assert parsed_schema["type"] == "object"
+    assert "name" in parsed_schema["properties"]
+    assert "email" in parsed_schema["properties"]
+    assert "plan_interest" in parsed_schema["properties"]
+    assert parsed_schema["required"] == ["name", "email", "plan_interest"]
+    
+    # Verify other params are preserved
+    assert result["max_tokens"] == 1024
+    assert result["anthropic_version"] == "bedrock-2023-05-31"
+
+
+def test_output_format_conversion_with_string_content():
+    """
+    Test that output_format conversion works when message content is a string (not a list).
+    """
+    from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
+        AmazonAnthropicClaudeMessagesConfig,
+    )
+    
+    config = AmazonAnthropicClaudeMessagesConfig()
+    
+    # Test messages with string content
+    messages = [
+        {"role": "user", "content": "What is 2+2?"}
+    ]
+    
+    output_format_schema = {
+        "type": "object",
+        "properties": {
+            "result": {"type": "integer"}
+        }
+    }
+    
+    anthropic_messages_optional_request_params = {
+        "max_tokens": 100,
+        "output_format": {
+            "type": "json_schema",
+            "schema": output_format_schema
+        }
+    }
+    
+    # Transform the request
+    result = config.transform_anthropic_messages_request(
+        model="anthropic.claude-sonnet-4-20250514-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+        litellm_params={},
+        headers={},
+    )
+    
+    # Verify the content was converted to list format
+    last_user_message = result["messages"][0]
+    content = last_user_message["content"]
+    assert isinstance(content, list), "content should be converted to list"
+    assert len(content) == 2, "content should have 2 items"
+    
+    # Check original text
+    assert content[0]["type"] == "text"
+    assert content[0]["text"] == "What is 2+2?"
+    
+    # Check schema was added
+    assert content[1]["type"] == "text"
+    parsed_schema = json.loads(content[1]["text"])
+    assert "result" in parsed_schema["properties"]
+
+
+def test_output_format_with_no_schema():
+    """
+    Test that if output_format has no schema, the conversion is skipped gracefully.
+    """
+    from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
+        AmazonAnthropicClaudeMessagesConfig,
+    )
+    
+    config = AmazonAnthropicClaudeMessagesConfig()
+    
+    messages = [
+        {"role": "user", "content": "Hello"}
+    ]
+    
+    anthropic_messages_optional_request_params = {
+        "max_tokens": 100,
+        "output_format": {
+            "type": "json_schema"
+            # No schema field
+        }
+    }
+    
+    # Transform the request
+    result = config.transform_anthropic_messages_request(
+        model="anthropic.claude-sonnet-4-20250514-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+        litellm_params={},
+        headers={},
+    )
+    
+    # Verify output_format was removed but no schema was added
+    assert "output_format" not in result
+    last_user_message = result["messages"][0]
+    
+    # Content should remain as string (not converted to list)
+    assert isinstance(last_user_message["content"], str)
+    assert last_user_message["content"] == "Hello"