From 9380f5ea011de363c4bcc250bd08c2ff3dce2a6c Mon Sep 17 00:00:00 2001 From: Mikalai Karytka Date: Mon, 26 Oct 2026 10:24:58 +0100 Subject: [PATCH] fix(anthropic): send extra_body fields at the top level and forward the server-side fallback beta extra_body passed to litellm.completion for an Anthropic model reached the Messages API as a literal nested "extra_body" key, and Anthropic rejected the request with "extra_body: Extra inputs are not permitted". That blocked every Anthropic request field litellm doesn't model yet, including fallbacks AnthropicConfig.transform_request now pops extra_body from the request params and merges its keys into the wire body. The direct Anthropic, Vertex AI and Azure paths all go through this transform. Paths on llm_http_handler, like Bedrock invoke, already pop extra_body before transform_request, so nothing is merged twice. EXTRA_BODY_ROUTING_KEYS is filtered out the same way as on the OpenAI-compatible path, so extra_body can't change the requested model fallbacks also needs the server-side-fallback-2026-07-01 beta, which the beta filter dropped as unknown, so Anthropic still answered "fallbacks: Extra inputs are not permitted". The beta is now forwarded for the anthropic provider only, since Anthropic documents server-side fallback for its own API. The Rust AnthropicBeta table mirrors the JSON config fallbacks is also registered in BETA_GATED_FIELDS, so under drop_params a request that doesn't end up sending the fallback beta, like a Vertex AI leg or a deployment without the header, drops fallbacks with a warning instead of getting a 400 The regression tests drive litellm.completion end to end against a mock transport and assert the request body --- .../src/providers/anthropic/beta.rs | 10 +- litellm/anthropic_beta_gated_fields.py | 14 ++- litellm/anthropic_beta_headers_config.json | 3 +- litellm/llms/anthropic/chat/transformation.py | 4 + .../chat/test_anthropic_chat_handler.py | 100 ++++++++++++++++++ .../test_anthropic_beta_headers_filtering.py | 19 ++++ 6 files changed, 142 insertions(+), 8 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 80095cac36c..aaff71a3622 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -124,12 +124,14 @@ pub enum AnthropicBeta { ToolSearchTool20251019, #[strum(serialize = "inline-tools-2026-09-15")] InlineTools20260915, + #[strum(serialize = "server-side-fallback-2026-07-01")] + ServerSideFallback20260701, #[strum(default, transparent)] Other(String), } impl AnthropicBeta { - pub const KNOWN: [Self; 40] = [ + pub const KNOWN: [Self; 41] = [ Self::Oauth20250420, Self::WebFetch20250910, Self::WebSearch20250305, @@ -170,6 +172,7 @@ impl AnthropicBeta { Self::ToolExamples20251029, Self::ToolSearchTool20251019, Self::InlineTools20260915, + Self::ServerSideFallback20260701, ]; pub fn as_str(&self) -> &str { @@ -242,7 +245,7 @@ impl AnthropicBeta { | Self::ThinkingDisplayUpdates20260818 => &[], Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse], Self::Effort20251124 => &[BetaProvider::VertexAi], - Self::FastMode20260201 | Self::Oauth20250420 => &[ + Self::FastMode20260201 | Self::Oauth20250420 | Self::ServerSideFallback20260701 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::Bedrock, @@ -441,7 +444,8 @@ mod tests { AnthropicBeta::TokenEfficientTools20250219, AnthropicBeta::ToolExamples20251029, AnthropicBeta::ToolSearchTool20251019, - AnthropicBeta::InlineTools20260915 + AnthropicBeta::InlineTools20260915, + AnthropicBeta::ServerSideFallback20260701 )] beta: AnthropicBeta, ) { diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py index fc36699bc93..7d097306178 100644 --- a/litellm/anthropic_beta_gated_fields.py +++ b/litellm/anthropic_beta_gated_fields.py @@ -29,6 +29,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" +ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: Final = "server-side-fallback-2026-07-01" THINKING_BLOCK_BINDING: Final = "thinking.block_binding" THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" @@ -36,6 +37,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" +FALLBACKS: Final = "fallbacks" _TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) @@ -48,6 +50,7 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: (FALLBACKS,), } ) @@ -227,8 +230,8 @@ def strip_beta_gated_system_messages( def strip_beta_gated_request_fields( request: Mapping[str, object], active_betas: Set[str], *, drop_params: object ) -> StrippedRequest: - """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when - nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + """Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped.""" thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) messages: Final = request.get("messages") message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None @@ -237,11 +240,14 @@ def strip_beta_gated_request_fields( if message_sequence is not None else StrippedMessages(messages=(), stripped=()) ) - stripped: Final = (*thinking.stripped, *system_messages.stripped) + fallbacks_gated: Final = ( + FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ())) if not stripped: return StrippedRequest(request=request, stripped=()) rewritten: Final = { - **request, + **{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)}, **({"thinking": thinking.thinking} if thinking.stripped else {}), **({"messages": system_messages.messages} if system_messages.stripped else {}), } diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index cee5ce15469..cbf3a5efaad 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -37,7 +37,8 @@ "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", - "inline-tools-2026-09-15": "inline-tools-2026-09-15" + "inline-tools-2026-09-15": "inline-tools-2026-09-15", + "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01" }, "azure_ai": { "advisor-tool-2026-03-01": null, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..d433da25cd7 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -106,6 +106,7 @@ from litellm.types.utils import ( ) from litellm.types.utils import Message as LitellmMessage from litellm.utils import ( + EXTRA_BODY_ROUTING_KEYS, ModelResponse, Usage, add_dummy_tool, @@ -166,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_EXTRA_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2419,6 +2421,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature dict[str, object], optional_params ) + extra_body: Final = _EXTRA_BODY_ADAPTER.validate_python(request_params.pop("extra_body", None) or {}) marked_tools: Final = tools_marked_for_a_tool_config_point( request_params.get("tools"), request_params.pop("cache_control_injection_points", None) ) @@ -2456,6 +2459,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "messages": anthropic_messages, **optional_params, **({} if marked_tools is None else {"tools": list(marked_tools)}), + **{k: v for k, v in extra_body.items() if k not in EXTRA_BODY_ROUTING_KEYS}, } self._apply_output_config(data=data, model=model, optional_params=optional_params) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 5d3fc523c0c..54777d467bb 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -136,6 +136,106 @@ def test_anthropic_completion_does_not_send_deployment_default_limits(): assert "default_api_key_tpm_limit" not in request_body +def test_anthropic_completion_sends_extra_body_fields_top_level_without_redirecting_model(): + captured_requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_extra_body", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={"fallbacks": [{"model": "claude-opus-5-5"}], "model": "claude-opus-5-5"}, + ) + finally: + client.close() + + request_body = json.loads(captured_requests[0].content) + assert request_body == { + "model": "claude-sonnet-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 16, + "fallbacks": [{"model": "claude-opus-5-5"}], + }, request_body + + +def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request: + captured_requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_fallbacks", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={"fallbacks": "default"}, + extra_headers=extra_headers, + drop_params=drop_params, + ) + finally: + client.close() + return captured_requests[0] + + +@pytest.mark.parametrize( + ("extra_headers", "drop_params", "expected_fallbacks"), + [ + ({"anthropic-beta": "server-side-fallback-2026-07-01"}, True, "default"), + (None, False, "default"), + (None, True, None), + ], +) +def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_the_fallback_beta( + extra_headers: dict[str, str] | None, + drop_params: bool, + expected_fallbacks: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _anthropic_request_with_fallbacks(extra_headers, drop_params) + assert json.loads(request.content).get("fallbacks") == expected_fallbacks + assert ("server-side-fallback-2026-07-01" in request.headers.get("anthropic-beta", "")) == ( + extra_headers is not None + ) + + def test_anthropic_tool_history_without_tools_injects_a_dummy_tool( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index deba7fe9587..e40a763e7ff 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -474,6 +474,25 @@ class TestAnthropicBetaHeadersFiltering: assert filtered == ["inline-tools-2026-09-15"] + @pytest.mark.parametrize( + ("provider", "expected"), + [ + ("anthropic", ["server-side-fallback-2026-07-01"]), + ("vertex_ai", []), + ("bedrock", []), + ], + ) + def test_server_side_fallback_forwarded_only_to_the_claude_api(self, provider: str, expected: list[str]) -> None: + """Anthropic rejects the `fallbacks` request field with 400 "fallbacks: Extra inputs are not permitted" unless + this beta is sent, and documents it for the Claude API only + (https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback, fetched 2026-10-26)""" + filtered: Final = filter_and_transform_beta_headers( + beta_headers=["server-side-fallback-2026-07-01"], + provider=provider, + ) + + assert filtered == expected + def test_null_value_headers_filtered(self): """Test that headers with null values are always filtered out.""" for provider in [