mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
fix(anthropic): forward service_tier on /v1/messages for anthropic, azure_ai and bedrock_mantle (#46370)
* test(integration): add known-bug translation cases for request fields LiteLLM drops service_tier on /v1/messages, conversation on /v1/responses and metadata on /v1/chat/completions never reach the provider. Each gets a TranslationTestCase with the correct provider body, marked xfail(strict=True) through the new TranslationTestCase.known_bug field, plus a passing service_tier case for /v1/messages on the openai route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): widen the known-bug cases to azure_ai, bedrock_mantle, openai/responses and azure deployments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): forward service_tier on /v1/messages for anthropic, azure_ai and bedrock_mantle Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
df3b44dbee
commit
542b254ca2
11 changed files with 199 additions and 6 deletions
|
|
@ -294,7 +294,6 @@ DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING: Final = (
|
|||
DROP_UNSUPPORTED_SPEED_WARNING: Final = (
|
||||
"Dropping unsupported `speed` for model=%s (drop_params=True). Fast mode is only supported on select Opus models."
|
||||
)
|
||||
|
||||
SERVICE_TIER_PROVIDERS: Final = frozenset({"anthropic", "azure_ai"})
|
||||
|
||||
OPENAI_TO_ANTHROPIC_SERVICE_TIER: Final = MappingProxyType(
|
||||
|
|
@ -306,6 +305,12 @@ DROP_UNSUPPORTED_SERVICE_TIER_WARNING: Final = (
|
|||
"Anthropic only accepts 'auto' and 'standard_only'."
|
||||
)
|
||||
|
||||
MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle"}
|
||||
|
||||
DROP_SERVICE_TIER_PROVIDER_WARNING: Final = (
|
||||
"Dropping `service_tier` for model=%s on provider=%s. Only the Anthropic API routes accept it."
|
||||
)
|
||||
|
||||
|
||||
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
||||
"""
|
||||
|
|
@ -539,6 +544,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
) -> None:
|
||||
return cls._maybe_drop_speed_param(model, optional_params, drop_params, custom_llm_provider)
|
||||
|
||||
@staticmethod
|
||||
def provider_supports_service_tier(custom_llm_provider: str | None) -> bool:
|
||||
"""Anthropic's ``service_tier`` is accepted by the direct API and the Foundry and Mantle gateways in
|
||||
front of it. Vertex rejects the field and Bedrock Invoke only takes its own tier names."""
|
||||
return custom_llm_provider is None or custom_llm_provider in MESSAGES_SERVICE_TIER_PROVIDERS
|
||||
|
||||
@classmethod
|
||||
def maybe_drop_service_tier_param(
|
||||
cls,
|
||||
model: str,
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> None:
|
||||
if "service_tier" not in optional_params or cls.provider_supports_service_tier(custom_llm_provider):
|
||||
return
|
||||
litellm.verbose_logger.warning(DROP_SERVICE_TIER_PROVIDER_WARNING, model, custom_llm_provider)
|
||||
optional_params.pop("service_tier", None)
|
||||
|
||||
@staticmethod
|
||||
def _raise_invalid_reasoning_effort(model: str, value: object, llm_provider: str) -> NoReturn:
|
||||
"""Raise a ``BadRequestError`` for an unrecognised ``reasoning_effort``.
|
||||
|
|
@ -717,12 +740,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Already handled above
|
||||
continue
|
||||
|
||||
if key == "properties" and isinstance(value, dict):
|
||||
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in value.items()}
|
||||
if key in ("properties", "$defs") and isinstance(value, dict):
|
||||
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
|
||||
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
|
||||
elif key == "items" and isinstance(value, dict):
|
||||
result[key] = AnthropicConfig.filter_anthropic_output_schema(value)
|
||||
elif key == "$defs" and isinstance(value, dict):
|
||||
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in value.items()}
|
||||
nested_schema: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
|
||||
result[key] = AnthropicConfig.filter_anthropic_output_schema(nested_schema)
|
||||
elif key == "anyOf" and isinstance(value, list):
|
||||
result["anyOf"] = result.get("anyOf", []) + [
|
||||
AnthropicConfig.filter_anthropic_output_schema(item) for item in value
|
||||
|
|
|
|||
|
|
@ -98,6 +98,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
"service_tier",
|
||||
"output_config",
|
||||
"reasoning_effort",
|
||||
"safeguards",
|
||||
|
|
|
|||
|
|
@ -205,6 +205,11 @@ class AnthropicMessagesRequestUtils:
|
|||
drop_params=drop_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
AnthropicConfig.maybe_drop_service_tier_param(
|
||||
model=model,
|
||||
optional_params=filtered_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
for param in ("temperature", "top_p", "top_k"):
|
||||
if param in filtered_params:
|
||||
AnthropicModelInfo._apply_sampling_param( # pyright: ignore[reportPrivateUsage] # same gating the /chat/completions path applies; forking it would drift
|
||||
|
|
|
|||
|
|
@ -56,6 +56,10 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
|
|||
model ID in the request body (unlike Bedrock Invoke which puts it in the URL).
|
||||
"""
|
||||
|
||||
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = (
|
||||
AmazonAnthropicClaudeMessagesConfig.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS | {"service_tier"}
|
||||
)
|
||||
|
||||
@property
|
||||
def beta_headers_provider(self) -> str:
|
||||
return "bedrock_mantle"
|
||||
|
|
|
|||
|
|
@ -430,6 +430,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
|
|||
cache_control: dict[str, Any] | None # Automatic prompt caching
|
||||
reasoning_effort: str | None
|
||||
safeguards: ReadOnly[list[dict[str, object]] | None]
|
||||
service_tier: ReadOnly[str | None]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.anthropic import CLAUDE_SONNET_4_6_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_4_6_TEST_CASE,
|
||||
scenario="service_tier",
|
||||
litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "service_tier": "standard_only"},
|
||||
expected_provider_request={
|
||||
**CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request,
|
||||
"service_tier": "standard_only",
|
||||
},
|
||||
mock_provider_response={**CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011Cg1RG36pCcTLE26htFFqC"},
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response,
|
||||
"id": "msg_011Cg1RG36pCcTLE26htFFqC",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_messages_service_tier_anthropic(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.azure_ai import CLAUDE_SONNET_4_6_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_4_6_TEST_CASE,
|
||||
scenario="service_tier",
|
||||
litellm_request={**CLAUDE_SONNET_4_6_TEST_CASE.litellm_request, "service_tier": "standard_only"},
|
||||
expected_provider_request={
|
||||
**CLAUDE_SONNET_4_6_TEST_CASE.expected_provider_request,
|
||||
"service_tier": "standard_only",
|
||||
},
|
||||
mock_provider_response={**CLAUDE_SONNET_4_6_TEST_CASE.mock_provider_response, "id": "msg_011Cg1SnrstCdzZPwmTk2qQa"},
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_4_6_TEST_CASE.expected_litellm_response,
|
||||
"id": "msg_011Cg1SnrstCdzZPwmTk2qQa",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [CLAUDE_SONNET_4_6_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_messages_service_tier_azure_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.bedrock_mantle import CLAUDE_SONNET_5_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
CLAUDE_SONNET_5_SERVICE_TIER_TEST_CASE: Final = replace(
|
||||
CLAUDE_SONNET_5_TEST_CASE,
|
||||
scenario="service_tier",
|
||||
litellm_request={**CLAUDE_SONNET_5_TEST_CASE.litellm_request, "service_tier": "standard_only"},
|
||||
expected_provider_request={**CLAUDE_SONNET_5_TEST_CASE.expected_provider_request, "service_tier": "standard_only"},
|
||||
mock_provider_response={
|
||||
**CLAUDE_SONNET_5_TEST_CASE.mock_provider_response,
|
||||
"id": "msg_bdrk_6oqdfuf56lp47vwldyos5f7l4lyyot337z2cv3h3lbaj3a3ei7vq",
|
||||
},
|
||||
expected_litellm_response={
|
||||
**CLAUDE_SONNET_5_TEST_CASE.expected_litellm_response,
|
||||
"id": "msg_bdrk_6oqdfuf56lp47vwldyos5f7l4lyyot337z2cv3h3lbaj3a3ei7vq",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [CLAUDE_SONNET_5_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_messages_service_tier_bedrock_mantle(
|
||||
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
|
||||
) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.provider import SharedProvider
|
||||
from integration.translation.case import TranslationTestCase
|
||||
from integration.translation.messages.bases.openai import GPT_5_4_TEST_CASE
|
||||
from integration.translation.runner import assert_translation
|
||||
|
||||
GPT_5_4_SERVICE_TIER_TEST_CASE: Final = replace(
|
||||
GPT_5_4_TEST_CASE,
|
||||
scenario="service_tier",
|
||||
litellm_request={**GPT_5_4_TEST_CASE.litellm_request, "service_tier": "auto"},
|
||||
expected_provider_request={**GPT_5_4_TEST_CASE.expected_provider_request, "service_tier": "auto"},
|
||||
mock_provider_response={
|
||||
**GPT_5_4_TEST_CASE.mock_provider_response,
|
||||
"id": "resp_0738fe6d3b086cd1006acef6c23a1487d2860e02c53af945a6",
|
||||
"created_at": 1791948482,
|
||||
"access_programs": None,
|
||||
"completed_at": 1791948483,
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_0738fe6d3b086cd1006acef6c310cc87d283ecc2cc1d6ff29c",
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Hello."}],
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
}
|
||||
],
|
||||
},
|
||||
expected_litellm_response={
|
||||
**GPT_5_4_TEST_CASE.expected_litellm_response,
|
||||
"id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3NGI3ZDQzZjBmNjQ1MjlkYWY4NWQ4MjRmNjE3ZGYzNDRjZTdhNGI3NjI3YTYzMDMzZGUzMDVmNTE2NTRlZjc2O3Jlc3BvbnNlX2lkOnJlc3BfMDczOGZlNmQzYjA4NmNkMTAwNmFjZWY2YzIzYTE0ODdkMjg2MGUwMmM1M2FmOTQ1YTY=",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [GPT_5_4_SERVICE_TIER_TEST_CASE], ids=lambda case: case.id)
|
||||
def test_messages_service_tier_openai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
|
||||
assert_translation(case, gateway, provider)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils
|
||||
|
||||
REQUEST: Final[dict[str, object]] = {
|
||||
"max_tokens": 64,
|
||||
"service_tier": "standard_only",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["anthropic", "azure_ai", "bedrock_mantle", None])
|
||||
def test_messages_forwards_service_tier_on_anthropic_api_routes(custom_llm_provider: str | None) -> None:
|
||||
optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
params=dict(REQUEST), model="claude-sonnet-4-6", drop_params=False, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
assert dict(optional_params).get("service_tier") == "standard_only"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "bedrock"])
|
||||
def test_messages_drops_service_tier_where_the_provider_rejects_it(custom_llm_provider: str) -> None:
|
||||
optional_params = AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
|
||||
params=dict(REQUEST), model="claude-sonnet-4-6", drop_params=False, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
assert "service_tier" not in optional_params
|
||||
assert dict(optional_params).get("max_tokens") == 64
|
||||
Loading…
Add table
Reference in a new issue