From 802031bc22c92d831c52ce0b0fa108cf3a26f31b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Wed, 21 Jan 2026 10:19:44 -0800 Subject: [PATCH] test_structured_output_e2e --- .../__init__.py | 12 +++ ...thropic_messages_structured_output_test.py | 98 +++++++++++++++++++ .../test_anthropic_api_structured_output.py | 29 ++++++ .../test_azure_anthropic_structured_output.py | 29 ++++++ ...test_bedrock_converse_structured_output.py | 29 ++++++ .../test_bedrock_invoke_structured_output.py | 29 ++++++ 6 files changed, 226 insertions(+) create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/__init__.py create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py create mode 100644 tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/__init__.py b/tests/pass_through_unit_tests/messages_api_structured_output/__init__.py new file mode 100644 index 00000000000..88c85d408a2 --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/__init__.py @@ -0,0 +1,12 @@ +""" +Anthropic Messages API Structured Outputs Test Suite + +E2E tests for structured outputs functionality across different providers: +- Direct Anthropic API +- Azure AI Foundry Anthropic models +- AWS Bedrock Invoke API +- AWS Bedrock Converse API + +All tests validate that the output_format parameter works correctly +and returns valid JSON instead of Markdown text. +""" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py b/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py new file mode 100644 index 00000000000..79d6324a786 --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py @@ -0,0 +1,98 @@ +""" +Base test class for Anthropic Messages API structured outputs E2E tests. + +Tests that structured outputs work correctly via litellm.anthropic.messages interface +by making actual API calls and validating JSON response format. +""" + +import json +import os +import sys +from abc import ABC, abstractmethod +from typing import Any, Dict, List + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest +import litellm + + +class BaseAnthropicMessagesStructuredOutputTest(ABC): + """ + Base test class for structured outputs E2E tests across different providers. + + Subclasses must implement: + - get_model(): Returns the model string to use for tests + """ + + @abstractmethod + def get_model(self) -> str: + """ + Returns the model string to use for tests. + """ + pass + + def get_output_format_schema(self) -> Dict[str, Any]: + """ + Returns a simple JSON schema for testing structured outputs. + """ + return { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "sentiment": { + "type": "string", + "enum": ["positive", "negative", "neutral"] + } + }, + "required": ["sentiment"], + "additionalProperties": False + } + } + + def get_test_messages(self) -> List[Dict[str, Any]]: + """ + Returns test messages for structured output testing. + """ + return [ + { + "role": "user", + "content": "What is the sentiment of this text: 'This product is amazing!' Return only the sentiment." + } + ] + + @pytest.mark.asyncio + async def test_structured_output_e2e(self): + """ + E2E test: Make actual API call with structured output and validate JSON response. + """ + messages = self.get_test_messages() + output_format = self.get_output_format_schema() + + response = await litellm.anthropic.messages.acreate( + model=self.get_model(), + messages=messages, + max_tokens=100, + output_format=output_format, + ) + + print(f"Response: {response}") + + # Validate response structure + assert "content" in response + assert len(response["content"]) > 0 + + content = response["content"][0] + assert "text" in content + + response_text = content["text"] + print(f"Response text: {response_text}") + + # The response should be valid JSON + parsed_json = json.loads(response_text) + print(f"Parsed JSON: {parsed_json}") + + # Validate the JSON structure + assert "sentiment" in parsed_json + assert parsed_json["sentiment"] in ["positive", "negative", "neutral"] \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py new file mode 100644 index 00000000000..6f3acd8232b --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py @@ -0,0 +1,29 @@ +""" +E2E Test suite for Anthropic API structured outputs via litellm.anthropic.messages. + +Tests that structured outputs work correctly with direct Anthropic API calls +by making actual API calls and validating JSON response format. + +Requires ANTHROPIC_API_KEY environment variable. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from .base_anthropic_messages_structured_output_test import ( + BaseAnthropicMessagesStructuredOutputTest, +) + + +class TestAnthropicAPIStructuredOutput(BaseAnthropicMessagesStructuredOutputTest): + """ + E2E tests for structured outputs with direct Anthropic API. + + Uses Claude Sonnet 4.5 which supports structured outputs with the + 'anthropic-beta: structured-outputs-2025-11-13' header. + """ + + def get_model(self) -> str: + return "claude-3-5-sonnet-20241022" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py new file mode 100644 index 00000000000..815e9a2e198 --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py @@ -0,0 +1,29 @@ +""" +E2E Test suite for Azure Anthropic structured outputs via litellm.anthropic.messages. + +Tests that structured outputs work correctly with Azure AI Foundry Anthropic models +by making actual API calls and validating JSON response format. + +Requires Azure AI credentials and model deployment. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from .base_anthropic_messages_structured_output_test import ( + BaseAnthropicMessagesStructuredOutputTest, +) + + +class TestAzureAnthropicStructuredOutput(BaseAnthropicMessagesStructuredOutputTest): + """ + E2E tests for structured outputs with Azure AI Foundry Anthropic models. + + Uses the azure_ai/ prefix which routes through Azure AI Foundry + while maintaining the Anthropic Messages API format. + """ + + def get_model(self) -> str: + return "azure_ai/claude-3-5-sonnet-20241022" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py new file mode 100644 index 00000000000..9229677f32c --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py @@ -0,0 +1,29 @@ +""" +E2E Test suite for Bedrock Converse API structured outputs via litellm.anthropic.messages. + +Tests that structured outputs work correctly with Bedrock Converse API +by making actual API calls and validating JSON response format. + +Requires AWS credentials and Bedrock model access. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from .base_anthropic_messages_structured_output_test import ( + BaseAnthropicMessagesStructuredOutputTest, +) + + +class TestBedrockConverseStructuredOutput(BaseAnthropicMessagesStructuredOutputTest): + """ + E2E tests for structured outputs with Bedrock Converse API. + + Uses the bedrock/converse/ prefix which routes through litellm.completion() + and the AmazonConverseConfig transformation. + """ + + def get_model(self) -> str: + return "bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py new file mode 100644 index 00000000000..2d37a3d42fe --- /dev/null +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py @@ -0,0 +1,29 @@ +""" +E2E Test suite for Bedrock Invoke API structured outputs via litellm.anthropic.messages. + +Tests that structured outputs work correctly with Bedrock Invoke API (native Anthropic format) +by making actual API calls and validating JSON response format. + +Requires AWS credentials and Bedrock model access. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from .base_anthropic_messages_structured_output_test import ( + BaseAnthropicMessagesStructuredOutputTest, +) + + +class TestBedrockInvokeStructuredOutput(BaseAnthropicMessagesStructuredOutputTest): + """ + E2E tests for structured outputs with Bedrock Invoke API. + + Uses the bedrock/invoke/ prefix which routes through the native + Anthropic Messages API format on Bedrock. + """ + + def get_model(self) -> str: + return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file