From 625b52cd658f1dc99d3884ecc59fc727ce9a4ced Mon Sep 17 00:00:00 2001 From: Mikalai Karytka Date: Mon, 26 Oct 2026 10:24:58 +0100 Subject: [PATCH] fix(anthropic): send extra_body fields at the top level of the Messages request extra_body passed to litellm.completion for an Anthropic model reached the Messages API as a literal nested "extra_body" key, and Anthropic rejected the request with "extra_body: Extra inputs are not permitted". That blocked every Anthropic request field litellm doesn't model yet, including server-side fallbacks AnthropicConfig.transform_request now pops extra_body from the request params and merges its keys into the wire body. The direct Anthropic, Vertex AI and Azure paths all go through this transform. Paths on llm_http_handler, like Bedrock invoke, already pop extra_body before transform_request, so nothing is merged twice. extra_body only adds fields the request doesn't already have. It can't override anything litellm built, can't change the model (EXTRA_BODY_ROUTING_KEYS), and can't set messages or system, so content that pre-call guardrails checked or redacted is the content that is sent fallbacks is registered in BETA_GATED_FIELDS under both betas that unlock it, server-side-fallback-2026-07-01 and server-side-fallback-2026-06-01. Under drop_params a request that doesn't end up sending either of them, like a Vertex AI leg where the 07-01 beta is filtered out or a deployment without the header, drops fallbacks with a warning instead of getting a 400 The proxy rejects fallbacks inside a client's extra_body with a 400. Anthropic runs the fallback model itself, so those targets never reach the key's model access checks. Server-side fallback is set in the deployment's litellm_params instead, next to the beta header the admin already configures there When Anthropic serves a turn from a fallback model, usage.iterations spans models, so its entries are no longer summed. The top-level usage, which Anthropic documents as the attempt that served the turn, is priced at the response's model. Attempts that declined with a billed refusal category are not priced yet The regression tests drive litellm.completion end to end against a mock transport and assert the request body --- litellm/anthropic_beta_gated_fields.py | 16 +- litellm/llms/anthropic/chat/transformation.py | 26 ++- litellm/proxy/auth/auth_utils.py | 9 + .../chat/test_anthropic_chat_handler.py | 162 ++++++++++++++++++ tests/unit/proxy/auth/test_auth_utils.py | 27 +++ 5 files changed, 232 insertions(+), 8 deletions(-) diff --git a/litellm/anthropic_beta_gated_fields.py b/litellm/anthropic_beta_gated_fields.py index fc36699bc93..17682ce1943 100644 --- a/litellm/anthropic_beta_gated_fields.py +++ b/litellm/anthropic_beta_gated_fields.py @@ -29,6 +29,8 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21" ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15" +ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: Final = "server-side-fallback-2026-06-01" +ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: Final = "server-side-fallback-2026-07-01" THINKING_BLOCK_BINDING: Final = "thinking.block_binding" THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates" @@ -36,6 +38,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at" SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config" SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal" SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)" +FALLBACKS: Final = "fallbacks" _TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"}) @@ -48,6 +51,8 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,), ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,), + ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: (FALLBACKS,), + ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: (FALLBACKS,), } ) @@ -227,8 +232,8 @@ def strip_beta_gated_system_messages( def strip_beta_gated_request_fields( request: Mapping[str, object], active_betas: Set[str], *, drop_params: object ) -> StrippedRequest: - """Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when - nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced.""" + """Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when + nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped.""" thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params) messages: Final = request.get("messages") message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None @@ -237,11 +242,14 @@ def strip_beta_gated_request_fields( if message_sequence is not None else StrippedMessages(messages=(), stripped=()) ) - stripped: Final = (*thinking.stripped, *system_messages.stripped) + fallbacks_gated: Final = ( + FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas) + ) + stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ())) if not stripped: return StrippedRequest(request=request, stripped=()) rewritten: Final = { - **request, + **{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)}, **({"thinking": thinking.thinking} if thinking.stripped else {}), **({"messages": system_messages.messages} if system_messages.stripped else {}), } diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2e908e66626..550d56d2bc4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -116,6 +116,7 @@ from litellm.types.utils import ( ) from litellm.types.utils import Message as LitellmMessage from litellm.utils import ( + EXTRA_BODY_ROUTING_KEYS, ModelResponse, Usage, add_dummy_tool, @@ -180,6 +181,7 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()}) _REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_EXTRA_BODY_GUARDED_KEYS: Final = EXTRA_BODY_ROUTING_KEYS | frozenset({"messages", "system"}) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -338,6 +340,7 @@ def _validated_defer_loading(value: object) -> bool: class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" + type: ReadOnly[str] input_tokens: ReadOnly[int | None] output_tokens: ReadOnly[int | None] cache_creation_input_tokens: ReadOnly[int | None] @@ -2544,6 +2547,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature dict[str, object], optional_params ) + extra_body: Final = _REQUEST_PARAMS_ADAPTER.validate_python(request_params.pop("extra_body", None) or {}) marked_tools: Final = tools_marked_for_a_tool_config_point( request_params.get("tools"), request_params.pop("cache_control_injection_points", None) ) @@ -2576,12 +2580,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ): request_params.pop("output_format") - data: Final = { + request_body: Final = { "model": model, "messages": anthropic_messages, - **optional_params, + **request_params, **({} if marked_tools is None else {"tools": list(marked_tools)}), } + withheld_extra_body_keys: Final = request_body.keys() | _EXTRA_BODY_GUARDED_KEYS + data: Final = { + **request_body, + **{k: v for k, v in extra_body.items() if k not in withheld_extra_body_keys}, + } self._apply_output_config(data=data, model=model, optional_params=optional_params) @@ -2854,9 +2863,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ephemeral_1h_input_tokens=detailed_1h, ) + @staticmethod + def _summable_iterations(usage: Mapping[str, Any]) -> Sequence[_AnthropicUsageIteration] | None: + """``usage.iterations`` when every entry ran on the response's model. A server-side fallback chain spans + models, and its top-level usage already is the attempt that served the turn, so it is never summed.""" + iterations: Final[Sequence[_AnthropicUsageIteration] | None] = usage.get("iterations") + if not iterations or any(iteration.get("type") == "fallback_message" for iteration in iterations): + return None + return iterations + @staticmethod def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None: - iterations: Final = usage.get("iterations") + iterations: Final = AnthropicConfig._summable_iterations(usage) if iterations: aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations) if aggregated is not None: @@ -2897,7 +2915,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): raw_speed: Final = _usage.get("speed") resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed - iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations") + iterations: Final = self._summable_iterations(_usage) if iterations: prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations) completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9350c9c935c..658b0b73720 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -703,6 +703,7 @@ def is_request_body_safe( nested = _coerce_metadata_to_dict(request_body.get(nested_key)) if nested is not None: _check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments) + _reject_client_chosen_fallbacks(nested_key, nested) for metadata_key in _NESTED_METADATA_KEYS: metadata = _coerce_metadata_to_dict(request_body.get(metadata_key)) if metadata is not None: @@ -738,6 +739,14 @@ def is_request_body_safe( return True +def _reject_client_chosen_fallbacks(nested_key: str, nested: Mapping[str, object]) -> None: + if nested_key == "extra_body" and "fallbacks" in nested: + raise ValueError( + "Rejected Request: fallbacks inside extra_body can't be checked against the key's allowed models. " + "Set server-side fallbacks in the deployment's litellm_params instead." + ) + + def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: reject_server_owned_wif_fields( diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index ce14adab4a0..6f0e96683c5 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -136,6 +136,168 @@ def test_anthropic_completion_does_not_send_deployment_default_limits(): assert "default_api_key_tpm_limit" not in request_body +def test_anthropic_completion_adds_extra_body_fields_top_level_without_overriding_the_request(): + captured_requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_extra_body", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={ + "fallbacks": [{"model": "claude-opus-5-5"}], + "model": "claude-opus-5-5", + "messages": [{"role": "user", "content": "unchecked"}], + "system": "unchecked", + "max_tokens": 4096, + }, + ) + finally: + client.close() + + request_body = json.loads(captured_requests[0].content) + assert request_body == { + "model": "claude-sonnet-5-5", + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}], + "max_tokens": 16, + "fallbacks": [{"model": "claude-opus-5-5"}], + }, request_body + + +def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request: + captured_requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "msg_fallbacks", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + litellm.completion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=16, + extra_body={"fallbacks": [{"model": "claude-opus-5-5"}]}, + extra_headers=extra_headers, + drop_params=drop_params, + ) + finally: + client.close() + return captured_requests[0] + + +@pytest.mark.parametrize( + ("fallback_beta", "drop_params", "fallbacks_kept"), + [ + ("server-side-fallback-2026-07-01", True, True), + ("server-side-fallback-2026-06-01", True, True), + (None, False, True), + (None, True, False), + ], +) +def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_a_fallback_beta( + fallback_beta: str | None, + drop_params: bool, + fallbacks_kept: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Both betas unlock ``fallbacks`` per https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback + (fetched 2026-11-03)""" + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true") + monkeypatch.setattr(litellm, "drop_params", False) + request: Final = _anthropic_request_with_fallbacks( + None if fallback_beta is None else {"anthropic-beta": fallback_beta}, drop_params + ) + sent_betas: Final = request.headers.get("anthropic-beta", "").split(",") + assert json.loads(request.content).get("fallbacks") == ([{"model": "claude-opus-5-5"}] if fallbacks_kept else None) + assert fallback_beta is None or fallback_beta in sent_betas + + +def test_anthropic_chat_prices_a_server_side_fallback_answer_by_the_attempt_that_served_it() -> None: + """Response shape from https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback (fetched + 2026-11-03): top-level usage is the attempt that served the turn, usage.iterations records every attempt""" + served_by_fallback: Final = { + "id": "msg_01XFUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [ + { + "type": "fallback", + "from": {"model": "claude-fable-5"}, + "to": {"model": "claude-opus-4-8"}, + "trigger": {"type": "refusal", "category": "cyber"}, + }, + {"type": "text", "text": "Hi! How can I help you today?"}, + ], + "stop_reason": "end_turn", + "stop_details": None, + "usage": { + "input_tokens": 412, + "output_tokens": 264, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "iterations": [ + {"type": "message", "model": "claude-fable-5", "input_tokens": 535, "output_tokens": 0}, + {"type": "fallback_message", "model": "claude-opus-4-8", "input_tokens": 412, "output_tokens": 264}, + ], + }, + } + client: Final = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=served_by_fallback))) + ) + try: + response: Final = litellm.completion( + model="anthropic/claude-fable-5", + messages=[{"role": "user", "content": "Hello"}], + api_key="test-key", + client=client, + max_tokens=300, + ) + finally: + client.close() + prompt_cost, completion_cost = litellm.cost_per_token( + model="claude-opus-4-8", prompt_tokens=412, completion_tokens=264 + ) + + assert response.model == "claude-opus-4-8" + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (412, 264) + assert response._hidden_params["response_cost"] == pytest.approx(prompt_cost + completion_cost) + + def test_anthropic_tool_history_without_tools_injects_a_dummy_tool( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 928c73d4b04..cd96422f9fc 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3522,6 +3522,33 @@ class TestIsRequestBodySafeBlocksFallbackSmuggle: ) +@pytest.mark.parametrize( + "extra_body", + [ + {"fallbacks": "default"}, + {"fallbacks": [{"model": "claude-opus-5-5"}]}, + '{"fallbacks": [{"model": "claude-opus-5-5"}]}', + ], +) +def test_client_chosen_server_side_fallbacks_in_extra_body_are_rejected(extra_body: object) -> None: + with pytest.raises(ValueError, match="fallbacks inside extra_body"): + is_request_body_safe( + request_body={"model": "claude-sonnet-5-5", "extra_body": extra_body}, + general_settings={}, + llm_router=None, + model="claude-sonnet-5-5", + ) + + +def test_extra_body_without_fallbacks_is_allowed() -> None: + assert is_request_body_safe( + request_body={"model": "claude-sonnet-5-5", "extra_body": {"stop_sequences": ["5"]}}, + general_settings={}, + llm_router=None, + model="claude-sonnet-5-5", + ) + + class TestIsRequestBodySafeRejectsUrlValuedFallback: @pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"]) def test_url_valued_string_fallback_is_rejected(self, fallback_field):