From 215e530048ac2bc7110e9b98e9ffad2bee90ee60 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 26 Jan 2026 13:28:36 -0800 Subject: [PATCH] init test suite: --- ..._anthropic_messages_input_examples_test.py | 201 ++++++++++++++++++ .../test_bedrock_invoke_input_examples.py | 51 +++++ ...st_bedrock_invoke_input_examples_sonnet.py | 45 ++++ 3 files changed, 297 insertions(+) create mode 100644 tests/pass_through_unit_tests/anthropic_input_examples_test_suite/base_anthropic_messages_input_examples_test.py create mode 100644 tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples.py create mode 100644 tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples_sonnet.py diff --git a/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/base_anthropic_messages_input_examples_test.py b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/base_anthropic_messages_input_examples_test.py new file mode 100644 index 00000000000..943ab6dca4c --- /dev/null +++ b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/base_anthropic_messages_input_examples_test.py @@ -0,0 +1,201 @@ +""" +Base test class for Anthropic Messages API input_examples E2E tests. + +Tests that input_examples works correctly via litellm.anthropic.messages interface +by making actual API calls and validating that the beta headers are correctly passed. + +Supported providers: +- Anthropic API: advanced-tool-use-2025-11-20 +- Microsoft Foundry: advanced-tool-use-2025-11-20 +- Vertex AI: tool-examples-2025-10-29 (all models) +- Bedrock Invoke: tool-examples-2025-10-29 (Claude Opus 4.5 only) + +Reference: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#providing-tool-use-examples +""" + +import json +import os +import sys +from abc import ABC, abstractmethod +from typing import Any, Dict, List + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest +import litellm + + +def get_weather_tool_with_input_examples() -> Dict[str, Any]: + """ + Returns a tool with input_examples field. + This demonstrates how to provide examples for tool inputs. + """ + return { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature" + } + }, + "required": ["location"] + }, + "input_examples": [ + { + "location": "San Francisco, CA", + "unit": "fahrenheit" + }, + { + "location": "Tokyo, Japan", + "unit": "celsius" + }, + { + "location": "New York, NY" # unit is optional + } + ] + } + + +def get_calculate_tool_with_input_examples() -> Dict[str, Any]: + """ + Returns a calculator tool with input_examples field. + """ + return { + "name": "calculate", + "description": "Perform basic arithmetic operations", + "input_schema": { + "type": "object", + "properties": { + "operation": { + "type": "string", + "enum": ["add", "subtract", "multiply", "divide"], + "description": "The arithmetic operation to perform" + }, + "a": { + "type": "number", + "description": "First number" + }, + "b": { + "type": "number", + "description": "Second number" + } + }, + "required": ["operation", "a", "b"] + }, + "input_examples": [ + { + "operation": "add", + "a": 5, + "b": 3 + }, + { + "operation": "multiply", + "a": 10, + "b": 2 + } + ] + } + + +class BaseAnthropicMessagesInputExamplesTest(ABC): + """ + Base test class for input_examples E2E tests across different providers. + + Subclasses must implement: + - get_model(): Returns the model string to use for tests + - get_extra_headers(): Returns extra headers (including anthropic-beta) + + Tests validate that input_examples are correctly handled and the appropriate + beta headers are passed to downstream providers. + """ + + @abstractmethod + def get_model(self) -> str: + """ + Returns the model string to use for tests. + + Examples: + - "anthropic/claude-sonnet-4-5-20250929" + - "vertex_ai/claude-opus-4-5@20251101" + - "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0" + """ + pass + + @abstractmethod + def get_extra_headers(self) -> Dict[str, str]: + """ + Returns extra headers to pass with the request. + Includes the anthropic-beta header for input examples. + + Different providers use different beta headers: + - Anthropic API: "advanced-tool-use-2025-11-20" + - Bedrock: "tool-examples-2025-10-29" (auto-injected by LiteLLM) + - Vertex AI: "tool-examples-2025-10-29" (auto-injected by LiteLLM) + """ + pass + + def get_tools_with_input_examples(self) -> List[Dict[str, Any]]: + """ + Returns tools list with input_examples. + """ + return [ + get_weather_tool_with_input_examples(), + get_calculate_tool_with_input_examples() + ] + + @pytest.mark.asyncio + async def test_input_examples_with_calculation(self): + """ + E2E test: Input examples should work with different tool types. + + This validates that the model can use the calculate tool with + input_examples correctly. + """ + litellm._turn_on_debug() + + tools = self.get_tools_with_input_examples() + messages = [ + { + "role": "user", + "content": "What is 15 plus 27? Use the calculate tool." + } + ] + + response = await litellm.anthropic.messages.acreate( + model=self.get_model(), + messages=messages, + tools=tools, + max_tokens=1024, + extra_headers=self.get_extra_headers(), + ) + + print(f"Response: {json.dumps(response, indent=2, default=str)}") + + # Validate response + assert "content" in response, "Response should contain content" + + content = response.get("content", []) + tool_uses = [block for block in content if block.get("type") == "tool_use"] + + # If the model decides to use a tool, it should be calculate + if tool_uses: + tool_names = [t.get("name") for t in tool_uses] + print(f"Tools used: {tool_names}") + + # Check if calculate was used + calc_tool_uses = [t for t in tool_uses if t.get("name") == "calculate"] + if calc_tool_uses: + tool_input = calc_tool_uses[0].get("input", {}) + print(f"Calculate tool input: {tool_input}") + # Validate the input has required fields + assert "operation" in tool_input, "Tool input should have operation" + assert "a" in tool_input, "Tool input should have a" + assert "b" in tool_input, "Tool input should have b" diff --git a/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples.py b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples.py new file mode 100644 index 00000000000..c9733e97794 --- /dev/null +++ b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples.py @@ -0,0 +1,51 @@ +""" +E2E Test suite for Anthropic Messages API input_examples with Bedrock Invoke. + +Tests that input_examples works correctly via litellm.anthropic.messages interface +by making actual API calls to AWS Bedrock. + +Bedrock Invoke: +- Beta header: tool-examples-2025-10-29 (auto-injected by LiteLLM) +- Supported models: Claude Opus 4.5 only + +Reference: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#providing-tool-use-examples +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest +from tests.pass_through_unit_tests.anthropic_input_examples_test_suite.base_anthropic_messages_input_examples_test import ( + BaseAnthropicMessagesInputExamplesTest, +) + + +class TestBedrockInvokeInputExamples(BaseAnthropicMessagesInputExamplesTest): + """ + E2E tests for input_examples with Bedrock Invoke API. + + Uses the bedrock/invoke/ prefix which routes through the native + Anthropic Messages API format on Bedrock. + + Beta header: tool-examples-2025-10-29 (auto-injected by LiteLLM) + + Note: Input examples on Bedrock is only supported on Claude Opus 4.5. + """ + + def get_model(self) -> str: + """ + Use Claude Opus 4.5 which is the only model that supports input_examples on Bedrock. + """ + return "bedrock/invoke/us.anthropic.claude-opus-4-5-20251101-v1:0" + + def get_extra_headers(self) -> dict: + """ + For Bedrock, we don't need to pass the beta header explicitly. + LiteLLM will auto-inject the correct beta header (tool-examples-2025-10-29) + when it detects input_examples in the tools. + + However, we can optionally pass it to test that user-provided headers work. + """ + return {} diff --git a/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples_sonnet.py b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples_sonnet.py new file mode 100644 index 00000000000..615008c191a --- /dev/null +++ b/tests/pass_through_unit_tests/anthropic_input_examples_test_suite/test_bedrock_invoke_input_examples_sonnet.py @@ -0,0 +1,45 @@ +""" +E2E Test for Bedrock Invoke with Claude Sonnet 4.5 and input_examples. + +This test validates that LiteLLM correctly handles input_examples for Sonnet 4.5: +- Claude Code (or users) will send tools WITH input_examples +- LiteLLM should automatically REMOVE input_examples for non-Opus 4.5 models +- The request should succeed without errors + +This ensures compatibility when Claude Code sends input_examples to all models, +but Bedrock only supports them on Opus 4.5. + +Reference: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#providing-tool-use-examples +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest +from tests.pass_through_unit_tests.anthropic_input_examples_test_suite.base_anthropic_messages_input_examples_test import ( + BaseAnthropicMessagesInputExamplesTest, +) + + +class TestBedrockInvokeSonnetInputExamples(BaseAnthropicMessagesInputExamplesTest): + """ + E2E tests for Bedrock Invoke with Claude Sonnet 4.5. + + This test sends tools WITH input_examples (as Claude Code would), + and validates that LiteLLM automatically removes them for Sonnet 4.5 + since Bedrock only supports input_examples on Opus 4.5. + """ + + def get_model(self) -> str: + """ + Use Claude Sonnet 4.5. + """ + return "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + + def get_extra_headers(self) -> dict: + """ + No extra headers needed. + """ + return {}