diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 7abc13d5508..f4e50bf3acd 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -294,7 +294,6 @@ DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING: Final = ( DROP_UNSUPPORTED_SPEED_WARNING: Final = ( "Dropping unsupported `speed` for model=%s (drop_params=True). Fast mode is only supported on select Opus models." ) - SERVICE_TIER_PROVIDERS: Final = frozenset({"anthropic", "azure_ai"}) OPENAI_TO_ANTHROPIC_SERVICE_TIER: Final = MappingProxyType( @@ -306,6 +305,12 @@ DROP_UNSUPPORTED_SERVICE_TIER_WARNING: Final = ( "Anthropic only accepts 'auto' and 'standard_only'." ) +MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle"} + +DROP_SERVICE_TIER_PROVIDER_WARNING: Final = ( + "Dropping `service_tier` for model=%s on provider=%s. Only the Anthropic API routes accept it." +) + class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ @@ -539,6 +544,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) -> None: return cls._maybe_drop_speed_param(model, optional_params, drop_params, custom_llm_provider) + @staticmethod + def provider_supports_service_tier(custom_llm_provider: str | None) -> bool: + """Anthropic's ``service_tier`` is accepted by the direct API and the Foundry and Mantle gateways in + front of it. Vertex rejects the field and Bedrock Invoke only takes its own tier names.""" + return custom_llm_provider is None or custom_llm_provider in MESSAGES_SERVICE_TIER_PROVIDERS + + @classmethod + def maybe_drop_service_tier_param( + cls, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + custom_llm_provider: str | None = None, + ) -> None: + if "service_tier" not in optional_params or cls.provider_supports_service_tier(custom_llm_provider): + return + litellm.verbose_logger.warning(DROP_SERVICE_TIER_PROVIDER_WARNING, model, custom_llm_provider) + optional_params.pop("service_tier", None) + @staticmethod def _raise_invalid_reasoning_effort(model: str, value: object, llm_provider: str) -> NoReturn: """Raise a ``BadRequestError`` for an unrecognised ``reasoning_effort``. @@ -717,12 +740,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Already handled above continue - if key == "properties" and isinstance(value, dict): - result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in value.items()} + if key in ("properties", "$defs") and isinstance(value, dict): + nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict + result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()} elif key == "items" and isinstance(value, dict): - result[key] = AnthropicConfig.filter_anthropic_output_schema(value) - elif key == "$defs" and isinstance(value, dict): - result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in value.items()} + nested_schema: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict + result[key] = AnthropicConfig.filter_anthropic_output_schema(nested_schema) elif key == "anyOf" and isinstance(value, list): result["anyOf"] = result.get("anyOf", []) + [ AnthropicConfig.filter_anthropic_output_schema(item) for item in value diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index f1eb30b63c1..4c667574d40 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -98,6 +98,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "output_format", "inference_geo", "speed", + "service_tier", "output_config", "reasoning_effort", "safeguards", diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/pass_through/messages/utils.py index 8371615baea..36bd10ab8e6 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/pass_through/messages/utils.py @@ -205,6 +205,11 @@ class AnthropicMessagesRequestUtils: drop_params=drop_params, custom_llm_provider=custom_llm_provider, ) + AnthropicConfig.maybe_drop_service_tier_param( + model=model, + optional_params=filtered_params, + custom_llm_provider=custom_llm_provider, + ) for param in ("temperature", "top_p", "top_k"): if param in filtered_params: AnthropicModelInfo._apply_sampling_param( # pyright: ignore[reportPrivateUsage] # same gating the /chat/completions path applies; forking it would drift diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 0ee5dcbe557..ca93cc3e617 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -56,6 +56,10 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): model ID in the request body (unlike Bedrock Invoke which puts it in the URL). """ + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = ( + AmazonAnthropicClaudeMessagesConfig.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS | {"service_tier"} + ) + @property def beta_headers_provider(self) -> str: return "bedrock_mantle" diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 25f55ddb84b..712f41ecc61 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -430,6 +430,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): cache_control: dict[str, Any] | None # Automatic prompt caching reasoning_effort: str | None safeguards: ReadOnly[list[dict[str, object]] | None] + service_tier: ReadOnly[str | None] class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): diff --git a/tests/integration/translation/messages/service_tier/__init__.py b/tests/integration/translation/messages/service_tier/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py b/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py new file mode 100644 index 00000000000..de453564e15 --- /dev/null +++ b/tests/integration/translation/messages/service_tier/test_messages_service_tier_anthropic.py @@ -0,0 +1,29 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace( + CLAUDE_SONNET_4_6_TEST_CASE, + scenario="service_tier", + litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "service_tier": "standard_only"}, + expected_provider_request={ + **CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request, + "service_tier": "standard_only", + }, + mock_provider_response={**CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011Cg1RG36pCcTLE26htFFqC"}, + expected_litellm_response={ + **CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response, + "id": "msg_011Cg1RG36pCcTLE26htFFqC", + }, +) + + +@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id) +def test_messages_service_tier_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/service_tier/test_messages_service_tier_azure_ai.py b/tests/integration/translation/messages/service_tier/test_messages_service_tier_azure_ai.py new file mode 100644 index 00000000000..13bf9dd888f --- /dev/null +++ b/tests/integration/translation/messages/service_tier/test_messages_service_tier_azure_ai.py @@ -0,0 +1,29 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.azure_ai import CLAUDE_SONNET_4_6_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace( + CLAUDE_SONNET_4_6_TEST_CASE, + scenario="service_tier", + litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "service_tier": "standard_only"}, + expected_provider_request={ + **CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request, + "service_tier": "standard_only", + }, + mock_provider_response={**CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011Cg1SnrstCdzZPwmTk2qQa"}, + expected_litellm_response={ + **CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response, + "id": "msg_011Cg1SnrstCdzZPwmTk2qQa", + }, +) + + +@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id) +def test_messages_service_tier_azure_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/service_tier/test_messages_service_tier_bedrock_mantle.py b/tests/integration/translation/messages/service_tier/test_messages_service_tier_bedrock_mantle.py new file mode 100644 index 00000000000..261d34271e9 --- /dev/null +++ b/tests/integration/translation/messages/service_tier/test_messages_service_tier_bedrock_mantle.py @@ -0,0 +1,31 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.bedrock_mantle import CLAUDE_SONNET_5_TEST_CASE +from integration.translation.runner import assert_translation + +CLAUDE_SONNET_5_SERVICE_TIER_TEST_CASE: Final = replace( + CLAUDE_SONNET_5_TEST_CASE, + scenario="service_tier", + litellm_request={**CLAUDE_SONNET_5_TEST_CASE.litellm_request, "service_tier": "standard_only"}, + expected_provider_request={**CLAUDE_SONNET_5_TEST_CASE.expected_provider_request, "service_tier": "standard_only"}, + mock_provider_response={ + **CLAUDE_SONNET_5_TEST_CASE.mock_provider_response, + "id": "msg_bdrk_6oqdfuf56lp47vwldyos5f7l4lyyot337z2cv3h3lbaj3a3ei7vq", + }, + expected_litellm_response={ + **CLAUDE_SONNET_5_TEST_CASE.expected_litellm_response, + "id": "msg_bdrk_6oqdfuf56lp47vwldyos5f7l4lyyot337z2cv3h3lbaj3a3ei7vq", + }, +) + + +@pytest.mark.parametrize("case", [CLAUDE_SONNET_5_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id) +def test_messages_service_tier_bedrock_mantle( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/service_tier/test_messages_service_tier_openai.py b/tests/integration/translation/messages/service_tier/test_messages_service_tier_openai.py new file mode 100644 index 00000000000..433c5bc4b2f --- /dev/null +++ b/tests/integration/translation/messages/service_tier/test_messages_service_tier_openai.py @@ -0,0 +1,42 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.openai import GPT_5_4_TEST_CASE +from integration.translation.runner import assert_translation + +GPT_5_4_SERVICE_TIER_TEST_CASE: Final = replace( + GPT_5_4_TEST_CASE, + scenario="service_tier", + litellm_request={**GPT_5_4_TEST_CASE.litellm_request, "service_tier": "auto"}, + expected_provider_request={**GPT_5_4_TEST_CASE.expected_provider_request, "service_tier": "auto"}, + mock_provider_response={ + **GPT_5_4_TEST_CASE.mock_provider_response, + "id": "resp_0738fe6d3b086cd1006acef6c23a1487d2860e02c53af945a6", + "created_at": 1791948482, + "access_programs": None, + "completed_at": 1791948483, + "output": [ + { + "id": "msg_0738fe6d3b086cd1006acef6c310cc87d283ecc2cc1d6ff29c", + "type": "message", + "status": "completed", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}], + "phase": "final_answer", + "role": "assistant", + } + ], + }, + expected_litellm_response={ + **GPT_5_4_TEST_CASE.expected_litellm_response, + "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3NGI3ZDQzZjBmNjQ1MjlkYWY4NWQ4MjRmNjE3ZGYzNDRjZTdhNGI3NjI3YTYzMDMzZGUzMDVmNTE2NTRlZjc2O3Jlc3BvbnNlX2lkOnJlc3BfMDczOGZlNmQzYjA4NmNkMTAwNmFjZWY2YzIzYTE0ODdkMjg2MGUwMmM1M2FmOTQ1YTY=", + }, +) + + +@pytest.mark.parametrize("case", [GPT_5_4_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id) +def test_messages_service_tier_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py new file mode 100644 index 00000000000..6472ccd1b2e --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py @@ -0,0 +1,28 @@ +from typing import Final + +import pytest + +from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils + +REQUEST: Final[dict[str, object]] = { + "max_tokens": 64, + "service_tier": "standard_only", + "messages": [{"role": "user", "content": "Hello"}], +} + + +@pytest.mark.parametrize("custom_llm_provider", ["anthropic", "azure_ai", "bedrock_mantle", None]) +def test_messages_forwards_service_tier_on_anthropic_api_routes(custom_llm_provider: str | None) -> None: + optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params=dict(REQUEST), model="claude-sonnet-4-6", drop_params=False, custom_llm_provider=custom_llm_provider + ) + assert dict(optional_params).get("service_tier") == "standard_only" + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "bedrock"]) +def test_messages_drops_service_tier_where_the_provider_rejects_it(custom_llm_provider: str) -> None: + optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( + params=dict(REQUEST), model="claude-sonnet-4-6", drop_params=False, custom_llm_provider=custom_llm_provider + ) + assert "service_tier" not in optional_params + assert dict(optional_params).get("max_tokens") == 64