test_structured_output_e2e

tests/pass_through_unit_tests/messages_api_structured_output/__init__.py

@@ -0,0 +1,12 @@
+"""
+Anthropic Messages API Structured Outputs Test Suite
+
+E2E tests for structured outputs functionality across different providers:
+- Direct Anthropic API
+- Azure AI Foundry Anthropic models
+- AWS Bedrock Invoke API
+- AWS Bedrock Converse API
+
+All tests validate that the output_format parameter works correctly
+and returns valid JSON instead of Markdown text.
+"""

tests/pass_through_unit_tests/messages_api_structured_output/base_anthropic_messages_structured_output_test.py

@@ -0,0 +1,98 @@
+"""
+Base test class for Anthropic Messages API structured outputs E2E tests.
+
+Tests that structured outputs work correctly via litellm.anthropic.messages interface
+by making actual API calls and validating JSON response format.
+"""
+
+import json
+import os
+import sys
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+sys.path.insert(0, os.path.abspath("../../.."))
+
+import pytest
+import litellm
+
+
+class BaseAnthropicMessagesStructuredOutputTest(ABC):
+    """
+    Base test class for structured outputs E2E tests across different providers.
+
+    Subclasses must implement:
+    - get_model(): Returns the model string to use for tests
+    """
+
+    @abstractmethod
+    def get_model(self) -> str:
+        """
+        Returns the model string to use for tests.
+        """
+        pass
+
+    def get_output_format_schema(self) -> Dict[str, Any]:
+        """
+        Returns a simple JSON schema for testing structured outputs.
+        """
+        return {
+            "type": "json_schema",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "sentiment": {
+                        "type": "string",
+                        "enum": ["positive", "negative", "neutral"]
+                    }
+                },
+                "required": ["sentiment"],
+                "additionalProperties": False
+            }
+        }
+
+    def get_test_messages(self) -> List[Dict[str, Any]]:
+        """
+        Returns test messages for structured output testing.
+        """
+        return [
+            {
+                "role": "user",
+                "content": "What is the sentiment of this text: 'This product is amazing!' Return only the sentiment."
+            }
+        ]
+
+    @pytest.mark.asyncio
+    async def test_structured_output_e2e(self):
+        """
+        E2E test: Make actual API call with structured output and validate JSON response.
+        """
+        messages = self.get_test_messages()
+        output_format = self.get_output_format_schema()
+
+        response = await litellm.anthropic.messages.acreate(
+            model=self.get_model(),
+            messages=messages,
+            max_tokens=100,
+            output_format=output_format,
+        )
+
+        print(f"Response: {response}")
+
+        # Validate response structure
+        assert "content" in response
+        assert len(response["content"]) > 0
+
+        content = response["content"][0]
+        assert "text" in content
+
+        response_text = content["text"]
+        print(f"Response text: {response_text}")
+
+        # The response should be valid JSON
+        parsed_json = json.loads(response_text)
+        print(f"Parsed JSON: {parsed_json}")
+
+        # Validate the JSON structure
+        assert "sentiment" in parsed_json
+        assert parsed_json["sentiment"] in ["positive", "negative", "neutral"]

tests/pass_through_unit_tests/messages_api_structured_output/test_anthropic_api_structured_output.py

@@ -0,0 +1,29 @@
+"""
+E2E Test suite for Anthropic API structured outputs via litellm.anthropic.messages.
+
+Tests that structured outputs work correctly with direct Anthropic API calls
+by making actual API calls and validating JSON response format.
+
+Requires ANTHROPIC_API_KEY environment variable.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+from .base_anthropic_messages_structured_output_test import (
+    BaseAnthropicMessagesStructuredOutputTest,
+)
+
+
+class TestAnthropicAPIStructuredOutput(BaseAnthropicMessagesStructuredOutputTest):
+    """
+    E2E tests for structured outputs with direct Anthropic API.
+
+    Uses Claude Sonnet 4.5 which supports structured outputs with the
+    'anthropic-beta: structured-outputs-2025-11-13' header.
+    """
+
+    def get_model(self) -> str:
+        return "claude-3-5-sonnet-20241022"

tests/pass_through_unit_tests/messages_api_structured_output/test_azure_anthropic_structured_output.py

@@ -0,0 +1,29 @@
+"""
+E2E Test suite for Azure Anthropic structured outputs via litellm.anthropic.messages.
+
+Tests that structured outputs work correctly with Azure AI Foundry Anthropic models
+by making actual API calls and validating JSON response format.
+
+Requires Azure AI credentials and model deployment.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+from .base_anthropic_messages_structured_output_test import (
+    BaseAnthropicMessagesStructuredOutputTest,
+)
+
+
+class TestAzureAnthropicStructuredOutput(BaseAnthropicMessagesStructuredOutputTest):
+    """
+    E2E tests for structured outputs with Azure AI Foundry Anthropic models.
+
+    Uses the azure_ai/ prefix which routes through Azure AI Foundry
+    while maintaining the Anthropic Messages API format.
+    """
+
+    def get_model(self) -> str:
+        return "azure_ai/claude-3-5-sonnet-20241022"

tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py

@@ -0,0 +1,29 @@
+"""
+E2E Test suite for Bedrock Converse API structured outputs via litellm.anthropic.messages.
+
+Tests that structured outputs work correctly with Bedrock Converse API
+by making actual API calls and validating JSON response format.
+
+Requires AWS credentials and Bedrock model access.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+from .base_anthropic_messages_structured_output_test import (
+    BaseAnthropicMessagesStructuredOutputTest,
+)
+
+
+class TestBedrockConverseStructuredOutput(BaseAnthropicMessagesStructuredOutputTest):
+    """
+    E2E tests for structured outputs with Bedrock Converse API.
+
+    Uses the bedrock/converse/ prefix which routes through litellm.completion()
+    and the AmazonConverseConfig transformation.
+    """
+
+    def get_model(self) -> str:
+        return "bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0"

tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py

@@ -0,0 +1,29 @@
+"""
+E2E Test suite for Bedrock Invoke API structured outputs via litellm.anthropic.messages.
+
+Tests that structured outputs work correctly with Bedrock Invoke API (native Anthropic format)
+by making actual API calls and validating JSON response format.
+
+Requires AWS credentials and Bedrock model access.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+from .base_anthropic_messages_structured_output_test import (
+    BaseAnthropicMessagesStructuredOutputTest,
+)
+
+
+class TestBedrockInvokeStructuredOutput(BaseAnthropicMessagesStructuredOutputTest):
+    """
+    E2E tests for structured outputs with Bedrock Invoke API.
+
+    Uses the bedrock/invoke/ prefix which routes through the native
+    Anthropic Messages API format on Bedrock.
+    """
+
+    def get_model(self) -> str:
+        return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0"