diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py index fc36699bc93..17682ce1943 100644 --- a/litellm/anthropic_beta_gated_fields.py +++ b/litellm/anthropic_beta_gated_fields.py @@ -29,6 +29,8 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" +ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: Final = "server-side-fallback-2026-06-01" +ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: Final = "server-side-fallback-2026-07-01" THINKING_BLOCK_BINDING: Final = "thinking.block_binding" THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" @@ -36,6 +38,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" +FALLBACKS: Final = "fallbacks" _TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) @@ -48,6 +51,8 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: (FALLBACKS,), + ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: (FALLBACKS,), } ) @@ -227,8 +232,8 @@ def strip_beta_gated_system_messages( def strip_beta_gated_request_fields( request: Mapping[str, object], active_betas: Set[str], *, drop_params: object ) -> StrippedRequest: - """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when - nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + """Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped.""" thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) messages: Final = request.get("messages") message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None @@ -237,11 +242,14 @@ def strip_beta_gated_request_fields( if message_sequence is not None else StrippedMessages(messages=(), stripped=()) ) - stripped: Final = (*thinking.stripped, *system_messages.stripped) + fallbacks_gated: Final = ( + FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ())) if not stripped: return StrippedRequest(request=request, stripped=()) rewritten: Final = { - **request, + **{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)}, **({"thinking": thinking.thinking} if thinking.stripped else {}), **({"messages": system_messages.messages} if system_messages.stripped else {}), } diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1977e3e2a7d..78b49fda0c4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -120,6 +120,7 @@ from litellm.types.utils import ( ) from litellm.types.utils import Message as LitellmMessage from litellm.utils import ( + EXTRA_BODY_ROUTING_KEYS, ModelResponse, Usage, add_dummy_tool, @@ -184,6 +185,7 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()}) _REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_EXTRA_BODY_GUARDED_KEYS: Final = EXTRA_BODY_ROUTING_KEYS | frozenset({"messages", "system"}) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2555,6 +2557,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature dict[str, object], optional_params ) + extra_body: Final = _REQUEST_PARAMS_ADAPTER.validate_python(request_params.pop("extra_body", None) or {}) marked_tools: Final = tools_marked_for_a_tool_config_point( request_params.get("tools"), request_params.pop("cache_control_injection_points", None) ) @@ -2587,12 +2590,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): request_params.pop("output_format") - data: Final = { + request_body: Final = { "model": model, "messages": anthropic_messages, - **optional_params, + **request_params, **({} if marked_tools is None else {"tools": list(marked_tools)}), } + withheld_extra_body_keys: Final = request_body.keys() | _EXTRA_BODY_GUARDED_KEYS + data: Final = { + **request_body, + **{k: v for k, v in extra_body.items() if k not in withheld_extra_body_keys}, + } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9350c9c935c..658b0b73720 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -703,6 +703,7 @@ def is_request_body_safe( nested = _coerce_metadata_to_dict(request_body.get(nested_key)) if nested is not None: _check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments) + _reject_client_chosen_fallbacks(nested_key, nested) for metadata_key in _NESTED_METADATA_KEYS: metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: @@ -738,6 +739,14 @@ def is_request_body_safe( return True +def _reject_client_chosen_fallbacks(nested_key: str, nested: Mapping[str, object]) -> None: + if nested_key == "extra_body" and "fallbacks" in nested: + raise ValueError( + "Rejected Request: fallbacks inside extra_body can't be checked against the key's allowed models. " + "Set server-side fallbacks in the deployment's litellm_params instead." + ) + + def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: reject_server_owned_wif_fields( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 7e2786327dc..b10a5085c6a 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -138,6 +138,116 @@ def test_anthropic_completion_does_not_send_deployment_default_limits(): assert "default_api_key_tpm_limit" not in request_body +def test_anthropic_completion_adds_extra_body_fields_top_level_without_overriding_the_request(): + captured_requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_extra_body", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={ + "fallbacks": [{"model": "claude-opus-5-5"}], + "model": "claude-opus-5-5", + "messages": [{"role": "user", "content": "unchecked"}], + "system": "unchecked", + "max_tokens": 4096, + }, + ) + finally: + client.close() + + request_body = json.loads(captured_requests[0].content) + assert request_body == { + "model": "claude-sonnet-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 16, + "fallbacks": [{"model": "claude-opus-5-5"}], + }, request_body + + +def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request: + captured_requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_fallbacks", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={"fallbacks": [{"model": "claude-opus-5-5"}]}, + extra_headers=extra_headers, + drop_params=drop_params, + ) + finally: + client.close() + return captured_requests[0] + + +@pytest.mark.parametrize( + ("fallback_beta", "drop_params", "fallbacks_kept"), + [ + ("server-side-fallback-2026-07-01", True, True), + ("server-side-fallback-2026-06-01", True, True), + (None, False, True), + (None, True, False), + ], +) +def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_a_fallback_beta( + fallback_beta: str | None, + drop_params: bool, + fallbacks_kept: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both betas unlock ``fallbacks`` per https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback + (fetched 2026-11-03)""" + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _anthropic_request_with_fallbacks( + None if fallback_beta is None else {"anthropic-beta": fallback_beta}, drop_params + ) + sent_betas: Final = request.headers.get("anthropic-beta", "").split(",") + assert json.loads(request.content).get("fallbacks") == ([{"model": "claude-opus-5-5"}] if fallbacks_kept else None) + assert fallback_beta is None or fallback_beta in sent_betas + + def test_anthropic_tool_history_without_tools_injects_a_dummy_tool( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 928c73d4b04..cd96422f9fc 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3522,6 +3522,33 @@ class TestIsRequestBodySafeBlocksFallbackSmuggle: ) +@pytest.mark.parametrize( + "extra_body", + [ + {"fallbacks": "default"}, + {"fallbacks": [{"model": "claude-opus-5-5"}]}, + '{"fallbacks": [{"model": "claude-opus-5-5"}]}', + ], +) +def test_client_chosen_server_side_fallbacks_in_extra_body_are_rejected(extra_body: object) -> None: + with pytest.raises(ValueError, match="fallbacks inside extra_body"): + is_request_body_safe( + request_body={"model": "claude-sonnet-5-5", "extra_body": extra_body}, + general_settings={}, + llm_router=None, + model="claude-sonnet-5-5", + ) + + +def test_extra_body_without_fallbacks_is_allowed() -> None: + assert is_request_body_safe( + request_body={"model": "claude-sonnet-5-5", "extra_body": {"stop_sequences": ["5"]}}, + general_settings={}, + llm_router=None, + model="claude-sonnet-5-5", + ) + + class TestIsRequestBodySafeRejectsUrlValuedFallback: @pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) def test_url_valued_string_fallback_is_rejected(self, fallback_field):