From 24faca9bcf5d8e2dfc5020c9abe8849d5221f53a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 22 Jan 2026 16:36:03 +0530 Subject: [PATCH] Add support for output formatfor bedrock invoke via v1/messages --- .../anthropic_unified/structured_output.md | 59 +++++- .../anthropic_claude3_transformation.py | 67 ++++++- ...ations_anthropic_claude3_transformation.py | 175 ++++++++++++++++++ 3 files changed, 297 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/anthropic_unified/structured_output.md b/docs/my-website/docs/anthropic_unified/structured_output.md index 433f57537dd..2a06cf82785 100644 --- a/docs/my-website/docs/anthropic_unified/structured_output.md +++ b/docs/my-website/docs/anthropic_unified/structured_output.md @@ -12,6 +12,7 @@ Use LiteLLM to call Anthropic's structured output feature via the `/v1/messages` | Anthropic | ✅ | Native support | | Azure AI (Anthropic models) | ✅ | Claude models on Azure AI | | Bedrock (Converse Anthropic models) | ✅ | Claude models via Bedrock Converse API | +| Bedrock (Invoke Anthropic models) | ✅ | Claude models via Bedrock Invoke API | ## Usage @@ -133,7 +134,7 @@ curl http://localhost:4000/v1/messages \ model_list: - model_name: bedrock-claude-sonnet litellm_params: - model: bedrock/anthropic.claude-sonnet-4-5-20250514-v1:0 + model: bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 @@ -178,6 +179,62 @@ curl http://localhost:4000/v1/messages \ }' ``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: bedrock-claude-invoke + litellm_params: + model: bedrock/invoke/global.anthropic.claude-sonnet-4-5-20250929-v1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-west-2 +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://localhost:4000/v1/messages \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -d '{ + "model": "bedrock-claude-invoke", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm." + } + ], + "output_format": { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + "plan_interest": {"type": "string"}, + "demo_requested": {"type": "boolean"} + }, + "required": ["name", "email", "plan_interest", "demo_requested"], + "additionalProperties": false + } + } + }' +``` + + diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 81ebb5a3601..a7065caece2 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -235,6 +235,63 @@ class AmazonAnthropicClaudeMessagesConfig( if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") + def _convert_output_format_to_inline_schema( + self, + output_format: Dict, + anthropic_messages_request: Dict, + ) -> None: + """ + Convert Anthropic output_format to inline schema in message content. + + Bedrock Invoke doesn't support the output_format parameter, so we embed + the schema directly into the user message content as text instructions. + + This approach adds the schema to the last user message, instructing the model + to respond in the specified JSON format. + + Args: + output_format: The output_format dict with 'type' and 'schema' + anthropic_messages_request: The request dict to modify in-place + + Ref: https://aws.amazon.com/blogs/machine-learning/structured-data-response-with-amazon-bedrock-prompt-engineering-and-tool-use/ + """ + import json + + # Extract schema from output_format + schema = output_format.get("schema") + if not schema: + return + + # Get messages from the request + messages = anthropic_messages_request.get("messages", []) + if not messages: + return + + # Find the last user message + last_user_message_idx = None + for idx in range(len(messages) - 1, -1, -1): + if messages[idx].get("role") == "user": + last_user_message_idx = idx + break + + if last_user_message_idx is None: + return + + last_user_message = messages[last_user_message_idx] + content = last_user_message.get("content", []) + + # Ensure content is a list + if isinstance(content, str): + content = [{"type": "text", "text": content}] + last_user_message["content"] = content + + # Add schema as text content to the message + schema_text = { + "type": "text", + "text": json.dumps(schema) + } + content.append(schema_text) + def transform_anthropic_messages_request( self, model: str, @@ -272,9 +329,13 @@ class AmazonAnthropicClaudeMessagesConfig( # 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it) self._remove_ttl_from_cache_control(anthropic_messages_request) - # 5. `output_format` is not supported on Bedrock invoke - if "output_format" in anthropic_messages_request: - anthropic_messages_request.pop("output_format", None) + # 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format) + output_format = anthropic_messages_request.pop("output_format", None) + if output_format: + self._convert_output_format_to_inline_schema( + output_format=output_format, + anthropic_messages_request=anthropic_messages_request, + ) # 6. AUTO-INJECT beta headers based on features used anthropic_model_info = AnthropicModelInfo() diff --git a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index a6f8a65f4ec..1cb84d32c1d 100644 --- a/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -104,3 +104,178 @@ def test_aws_params_filtered_from_request_body(): # Verify messages are present assert "messages" in result, "messages should be in request body" assert len(result["messages"]) == 1, "should have 1 message" + + +def test_output_format_conversion_to_inline_schema(): + """ + Test that output_format is converted to inline schema in message content for Bedrock Invoke. + + Bedrock Invoke doesn't support the output_format parameter, so LiteLLM converts it by + embedding the schema directly into the user message content. + """ + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + config = AmazonAnthropicClaudeMessagesConfig() + + # Test messages + messages = [ + {"role": "user", "content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan."} + ] + + # Output format with schema + output_format_schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + "plan_interest": {"type": "string"} + }, + "required": ["name", "email", "plan_interest"], + "additionalProperties": False + } + + anthropic_messages_optional_request_params = { + "max_tokens": 1024, + "output_format": { + "type": "json_schema", + "schema": output_format_schema + } + } + + # Transform the request + result = config.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-20250514-v1:0", + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params={}, + headers={}, + ) + + # Verify output_format was removed from the request + assert "output_format" not in result, "output_format should be removed from request body" + + # Verify the schema was added to the last user message content + assert "messages" in result + last_user_message = result["messages"][0] + assert last_user_message["role"] == "user" + + content = last_user_message["content"] + assert isinstance(content, list), "content should be a list" + assert len(content) == 2, "content should have 2 items (original text + schema)" + + # Check original text is preserved + assert content[0]["type"] == "text" + assert "John Smith" in content[0]["text"] + + # Check schema was added as JSON string + assert content[1]["type"] == "text" + schema_text = content[1]["text"] + + # Parse the schema JSON + parsed_schema = json.loads(schema_text) + assert parsed_schema["type"] == "object" + assert "name" in parsed_schema["properties"] + assert "email" in parsed_schema["properties"] + assert "plan_interest" in parsed_schema["properties"] + assert parsed_schema["required"] == ["name", "email", "plan_interest"] + + # Verify other params are preserved + assert result["max_tokens"] == 1024 + assert result["anthropic_version"] == "bedrock-2023-05-31" + + +def test_output_format_conversion_with_string_content(): + """ + Test that output_format conversion works when message content is a string (not a list). + """ + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + config = AmazonAnthropicClaudeMessagesConfig() + + # Test messages with string content + messages = [ + {"role": "user", "content": "What is 2+2?"} + ] + + output_format_schema = { + "type": "object", + "properties": { + "result": {"type": "integer"} + } + } + + anthropic_messages_optional_request_params = { + "max_tokens": 100, + "output_format": { + "type": "json_schema", + "schema": output_format_schema + } + } + + # Transform the request + result = config.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-20250514-v1:0", + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params={}, + headers={}, + ) + + # Verify the content was converted to list format + last_user_message = result["messages"][0] + content = last_user_message["content"] + assert isinstance(content, list), "content should be converted to list" + assert len(content) == 2, "content should have 2 items" + + # Check original text + assert content[0]["type"] == "text" + assert content[0]["text"] == "What is 2+2?" + + # Check schema was added + assert content[1]["type"] == "text" + parsed_schema = json.loads(content[1]["text"]) + assert "result" in parsed_schema["properties"] + + +def test_output_format_with_no_schema(): + """ + Test that if output_format has no schema, the conversion is skipped gracefully. + """ + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + + config = AmazonAnthropicClaudeMessagesConfig() + + messages = [ + {"role": "user", "content": "Hello"} + ] + + anthropic_messages_optional_request_params = { + "max_tokens": 100, + "output_format": { + "type": "json_schema" + # No schema field + } + } + + # Transform the request + result = config.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-20250514-v1:0", + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params={}, + headers={}, + ) + + # Verify output_format was removed but no schema was added + assert "output_format" not in result + last_user_message = result["messages"][0] + + # Content should remain as string (not converted to list) + assert isinstance(last_user_message["content"], str) + assert last_user_message["content"] == "Hello"