fix(anthropic): send extra_body fields at the top level and forward the server-side fallback beta

extra_body passed to litellm.completion for an Anthropic model reached the Messages API as a literal nested "extra_body" key, and Anthropic rejected the request with "extra_body: Extra inputs are not permitted". That blocked every Anthropic request field litellm doesn't model yet, including fallbacks AnthropicConfig.transform_request now pops extra_body from the request params and merges its keys into the wire body. The direct Anthropic, Vertex AI and Azure paths all go through this transform. Paths on llm_http_handler, like Bedrock invoke, already pop extra_body before transform_request, so nothing is merged twice. EXTRA_BODY_ROUTING_KEYS is filtered out the same way as on the OpenAI-compatible path, so extra_body can't change the requested model fallbacks also needs the server-side-fallback-2026-07-01 beta, which the beta filter dropped as unknown, so Anthropic still answered "fallbacks: Extra inputs are not permitted". The beta is now forwarded for the anthropic provider only, since Anthropic documents server-side fallback for its own API. The Rust AnthropicBeta table mirrors the JSON config fallbacks is also registered in BETA_GATED_FIELDS, so under drop_params a request that doesn't end up sending the fallback beta, like a Vertex AI leg or a deployment without the header, drops fallbacks with a warning instead of getting a 400 The regression tests drive litellm.completion end to end against a mock transport and assert the request body
2026-11-08 08:08:37 +00:00 · 2026-10-26 10:24:58 +01:00 · 2026-10-26 10:24:58 +01:00 · 9380f5ea01
commit 9380f5ea01
parent ccc56d27ad
6 changed files with 142 additions and 8 deletions
--- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
+++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
@ -124,12 +124,14 @@ pub enum AnthropicBeta {
    ToolSearchTool20251019,
    #[strum(serialize = "inline-tools-2026-09-15")]
    InlineTools20260915,
+    #[strum(serialize = "server-side-fallback-2026-07-01")]
+    ServerSideFallback20260701,
    #[strum(default, transparent)]
    Other(String),
 }

 impl AnthropicBeta {
-    pub const KNOWN: [Self; 40] = [
+    pub const KNOWN: [Self; 41] = [
        Self::Oauth20250420,
        Self::WebFetch20250910,
        Self::WebSearch20250305,
@ -170,6 +172,7 @@ impl AnthropicBeta {
        Self::ToolExamples20251029,
        Self::ToolSearchTool20251019,
        Self::InlineTools20260915,
+        Self::ServerSideFallback20260701,
    ];

    pub fn as_str(&self) -> &str {
@ -242,7 +245,7 @@ impl AnthropicBeta {
            | Self::ThinkingDisplayUpdates20260818 => &[],
            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
            Self::Effort20251124 => &[BetaProvider::VertexAi],
-            Self::FastMode20260201 | Self::Oauth20250420 => &[
+            Self::FastMode20260201 | Self::Oauth20250420 | Self::ServerSideFallback20260701 => &[
                BetaProvider::AzureAi,
                BetaProvider::BedrockConverse,
                BetaProvider::Bedrock,
@ -441,7 +444,8 @@ mod tests {
            AnthropicBeta::TokenEfficientTools20250219,
            AnthropicBeta::ToolExamples20251029,
            AnthropicBeta::ToolSearchTool20251019,
-            AnthropicBeta::InlineTools20260915
+            AnthropicBeta::InlineTools20260915,
+            AnthropicBeta::ServerSideFallback20260701
        )]
        beta: AnthropicBeta,
    ) {
--- a/litellm/anthropic_beta_gated_fields.py
+++ b/litellm/anthropic_beta_gated_fields.py
@ -29,6 +29,7 @@ from litellm.types.llms.anthropic import (

 ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
 ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
+ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: Final = "server-side-fallback-2026-07-01"

 THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
 THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
@ -36,6 +37,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
 SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
 SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
 SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
+FALLBACKS: Final = "fallbacks"

 _TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})

@ -48,6 +50,7 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
        ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
        ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
        ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
+        ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: (FALLBACKS,),
    }
 )

@ -227,8 +230,8 @@ def strip_beta_gated_system_messages(
 def strip_beta_gated_request_fields(
    request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
 ) -> StrippedRequest:
-    """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
-    nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
+    """Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
+    nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped."""
    thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
    messages: Final = request.get("messages")
    message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
@ -237,11 +240,14 @@ def strip_beta_gated_request_fields(
        if message_sequence is not None
        else StrippedMessages(messages=(), stripped=())
    )
-    stripped: Final = (*thinking.stripped, *system_messages.stripped)
+    fallbacks_gated: Final = (
+        FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas)
+    )
+    stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ()))
    if not stripped:
        return StrippedRequest(request=request, stripped=())
    rewritten: Final = {
-        **request,
+        **{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)},
        **({"thinking": thinking.thinking} if thinking.stripped else {}),
        **({"messages": system_messages.messages} if system_messages.stripped else {}),
    }
--- a/litellm/anthropic_beta_headers_config.json
+++ b/litellm/anthropic_beta_headers_config.json
@ -37,7 +37,8 @@
    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01"
  },
  "azure_ai": {
    "advisor-tool-2026-03-01": null,
--- a/litellm/llms/anthropic/chat/transformation.py
+++ b/litellm/llms/anthropic/chat/transformation.py
@ -106,6 +106,7 @@ from litellm.types.utils import (
 )
 from litellm.types.utils import Message as LitellmMessage
 from litellm.utils import (
+    EXTRA_BODY_ROUTING_KEYS,
    ModelResponse,
    Usage,
    add_dummy_tool,
@ -166,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
 _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _TOOLS_ADAPTER: Final = TypeAdapter(list[object])
 _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_EXTRA_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
    ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
 )
@ -2419,6 +2421,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
        request_params: Final = cast(  # cast-ok: optional_params is the untyped dict of the shared transform signature
            dict[str, object], optional_params
        )
+        extra_body: Final = _EXTRA_BODY_ADAPTER.validate_python(request_params.pop("extra_body", None) or {})
        marked_tools: Final = tools_marked_for_a_tool_config_point(
            request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
        )
@ -2456,6 +2459,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
            "messages": anthropic_messages,
            **optional_params,
            **({} if marked_tools is None else {"tools": list(marked_tools)}),
+            **{k: v for k, v in extra_body.items() if k not in EXTRA_BODY_ROUTING_KEYS},
        }

        self._apply_output_config(data=data, model=model, optional_params=optional_params)
--- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
+++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
@ -136,6 +136,106 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
    assert "default_api_key_tpm_limit" not in request_body


+def test_anthropic_completion_sends_extra_body_fields_top_level_without_redirecting_model():
+    captured_requests: list[httpx.Request] = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "msg_extra_body",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-5-5",
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+    try:
+        litellm.completion(
+            model="anthropic/claude-sonnet-5-5",
+            messages=[{"role": "user", "content": "Hello"}],
+            api_key="test-key",
+            client=client,
+            max_tokens=16,
+            extra_body={"fallbacks": [{"model": "claude-opus-5-5"}], "model": "claude-opus-5-5"},
+        )
+    finally:
+        client.close()
+
+    request_body = json.loads(captured_requests[0].content)
+    assert request_body == {
+        "model": "claude-sonnet-5-5",
+        "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
+        "max_tokens": 16,
+        "fallbacks": [{"model": "claude-opus-5-5"}],
+    }, request_body
+
+
+def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request:
+    captured_requests: Final[list[httpx.Request]] = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "msg_fallbacks",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-5-5",
+                "content": [{"type": "text", "text": "Hello"}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+
+    client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+    try:
+        litellm.completion(
+            model="anthropic/claude-sonnet-5-5",
+            messages=[{"role": "user", "content": "Hello"}],
+            api_key="test-key",
+            client=client,
+            max_tokens=16,
+            extra_body={"fallbacks": "default"},
+            extra_headers=extra_headers,
+            drop_params=drop_params,
+        )
+    finally:
+        client.close()
+    return captured_requests[0]
+
+
+@pytest.mark.parametrize(
+    ("extra_headers", "drop_params", "expected_fallbacks"),
+    [
+        ({"anthropic-beta": "server-side-fallback-2026-07-01"}, True, "default"),
+        (None, False, "default"),
+        (None, True, None),
+    ],
+)
+def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_the_fallback_beta(
+    extra_headers: dict[str, str] | None,
+    drop_params: bool,
+    expected_fallbacks: str | None,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
+    monkeypatch.setattr(litellm, "drop_params", False)
+    request: Final = _anthropic_request_with_fallbacks(extra_headers, drop_params)
+    assert json.loads(request.content).get("fallbacks") == expected_fallbacks
+    assert ("server-side-fallback-2026-07-01" in request.headers.get("anthropic-beta", "")) == (
+        extra_headers is not None
+    )
+
+
 def test_anthropic_tool_history_without_tools_injects_a_dummy_tool(
    respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
 ) -> None:
--- a/tests/unit/test_anthropic_beta_headers_filtering.py
+++ b/tests/unit/test_anthropic_beta_headers_filtering.py
@ -474,6 +474,25 @@ class TestAnthropicBetaHeadersFiltering:

        assert filtered == ["inline-tools-2026-09-15"]

+    @pytest.mark.parametrize(
+        ("provider", "expected"),
+        [
+            ("anthropic", ["server-side-fallback-2026-07-01"]),
+            ("vertex_ai", []),
+            ("bedrock", []),
+        ],
+    )
+    def test_server_side_fallback_forwarded_only_to_the_claude_api(self, provider: str, expected: list[str]) -> None:
+        """Anthropic rejects the `fallbacks` request field with 400 "fallbacks: Extra inputs are not permitted" unless
+        this beta is sent, and documents it for the Claude API only
+        (https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback, fetched 2026-10-26)"""
+        filtered: Final = filter_and_transform_beta_headers(
+            beta_headers=["server-side-fallback-2026-07-01"],
+            provider=provider,
+        )
+
+        assert filtered == expected
+
    def test_null_value_headers_filtered(self):
        """Test that headers with null values are always filtered out."""
        for provider in [