scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(anthropic): send extra_body fields at the top level of the Messages request

Browse source 
extra_body passed to litellm.completion for an Anthropic model reached the
Messages API as a literal nested "extra_body" key, and Anthropic rejected the
request with "extra_body: Extra inputs are not permitted". That blocked every
Anthropic request field litellm doesn't model yet, including server-side
fallbacks

AnthropicConfig.transform_request now pops extra_body from the request params
and merges its keys into the wire body. The direct Anthropic, Vertex AI and
Azure paths all go through this transform. Paths on llm_http_handler, like
Bedrock invoke, already pop extra_body before transform_request, so nothing
is merged twice. extra_body only adds fields the request doesn't already
have. It can't override anything litellm built, can't change the model
(EXTRA_BODY_ROUTING_KEYS), and can't set messages or system, so content
that pre-call guardrails checked or redacted is the content that is sent

fallbacks is registered in BETA_GATED_FIELDS under both betas that unlock it,
server-side-fallback-2026-07-01 and server-side-fallback-2026-06-01. Under
drop_params a request that doesn't end up sending either of them, like a
Vertex AI leg where the 07-01 beta is filtered out or a deployment without
the header, drops fallbacks with a warning instead of getting a 400

The proxy rejects fallbacks inside a client's extra_body with a 400. Anthropic
runs the fallback model itself, so those targets never reach the key's model
access checks. Server-side fallback is set in the deployment's litellm_params
instead, next to the beta header the admin already configures there

When Anthropic serves a turn from a fallback model, usage.iterations spans
models, so its entries are no longer summed. The top-level usage, which
Anthropic documents as the attempt that served the turn, is priced at the
response's model. Attempts that declined with a billed refusal category are
not priced yet

The regression tests drive litellm.completion end to end against a mock
transport and assert the request body
This commit is contained in:
Mikalai Karytka 2026-10-26 10:24:58 +01:00
parent b777dd6f14
commit 625b52cd65
No known key found for this signature in database
5 changed files with 232 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/anthropic_beta_gated_fields.py
View file

@ -29,6 +29,8 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: Final = "server-side-fallback-2026-06-01"
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: Final = "server-side-fallback-2026-07-01"
THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
@ -36,6 +38,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
FALLBACKS: Final = "fallbacks"
_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
@ -48,6 +51,8 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: (FALLBACKS,),
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: (FALLBACKS,),
}
)
@ -227,8 +232,8 @@ def strip_beta_gated_system_messages(
def strip_beta_gated_request_fields(
request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
) -> StrippedRequest:
"""Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
"""Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped."""
thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
messages: Final = request.get("messages")
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
@ -237,11 +242,14 @@ def strip_beta_gated_request_fields(
if message_sequence is not None
else StrippedMessages(messages=(), stripped=())
)
stripped: Final = (*thinking.stripped, *system_messages.stripped)
fallbacks_gated: Final = (
FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas)
)
stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ()))
if not stripped:
return StrippedRequest(request=request, stripped=())
rewritten: Final = {
**request,
**{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)},
**({"thinking": thinking.thinking} if thinking.stripped else {}),
**({"messages": system_messages.messages} if system_messages.stripped else {}),
}

26
litellm/llms/anthropic/chat/transformation.py
View file

@ -116,6 +116,7 @@ from litellm.types.utils import (
)
from litellm.types.utils import Message as LitellmMessage
from litellm.utils import (
EXTRA_BODY_ROUTING_KEYS,
ModelResponse,
Usage,
add_dummy_tool,
@ -180,6 +181,7 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_EXTRA_BODY_GUARDED_KEYS: Final = EXTRA_BODY_ROUTING_KEYS | frozenset({"messages", "system"})
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -338,6 +340,7 @@ def _validated_defer_loading(value: object) -> bool:
class _AnthropicUsageIteration(TypedDict, total=False):
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
type: ReadOnly[str]
input_tokens: ReadOnly[int | None]
output_tokens: ReadOnly[int | None]
cache_creation_input_tokens: ReadOnly[int | None]
@ -2544,6 +2547,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
dict[str, object], optional_params
)
extra_body: Final = _REQUEST_PARAMS_ADAPTER.validate_python(request_params.pop("extra_body", None) or {})
marked_tools: Final = tools_marked_for_a_tool_config_point(
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
)
@ -2576,12 +2580,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
):
request_params.pop("output_format")
data: Final = {
request_body: Final = {
"model": model,
"messages": anthropic_messages,
**optional_params,
**request_params,
**({} if marked_tools is None else {"tools": list(marked_tools)}),
}
withheld_extra_body_keys: Final = request_body.keys() | _EXTRA_BODY_GUARDED_KEYS
data: Final = {
**request_body,
**{k: v for k, v in extra_body.items() if k not in withheld_extra_body_keys},
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)
@ -2854,9 +2863,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ephemeral_1h_input_tokens=detailed_1h,
)
@staticmethod
def _summable_iterations(usage: Mapping[str, Any]) -> Sequence[_AnthropicUsageIteration] | None:
"""``usage.iterations`` when every entry ran on the response's model. A server-side fallback chain spans
models, and its top-level usage already is the attempt that served the turn, so it is never summed."""
iterations: Final[Sequence[_AnthropicUsageIteration] | None] = usage.get("iterations")
if not iterations or any(iteration.get("type") == "fallback_message" for iteration in iterations):
return None
return iterations
@staticmethod
def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None:
iterations: Final = usage.get("iterations")
iterations: Final = AnthropicConfig._summable_iterations(usage)
if iterations:
aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations)
if aggregated is not None:
@ -2897,7 +2915,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
raw_speed: Final = _usage.get("speed")
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations")
iterations: Final = self._summable_iterations(_usage)
if iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations)

9
litellm/proxy/auth/auth_utils.py
View file

@ -703,6 +703,7 @@ def is_request_body_safe(
nested = _coerce_metadata_to_dict(request_body.get(nested_key))
if nested is not None:
_check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments)
_reject_client_chosen_fallbacks(nested_key, nested)
for metadata_key in _NESTED_METADATA_KEYS:
metadata = _coerce_metadata_to_dict(request_body.get(metadata_key))
if metadata is not None:
@ -738,6 +739,14 @@ def is_request_body_safe(
return True
def _reject_client_chosen_fallbacks(nested_key: str, nested: Mapping[str, object]) -> None:
if nested_key == "extra_body" and "fallbacks" in nested:
raise ValueError(
"Rejected Request: fallbacks inside extra_body can't be checked against the key's allowed models. "
"Set server-side fallbacks in the deployment's litellm_params instead."
)
def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
if manages_deployments:
reject_server_owned_wif_fields(

162
tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -136,6 +136,168 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
assert "default_api_key_tpm_limit" not in request_body
def test_anthropic_completion_adds_extra_body_fields_top_level_without_overriding_the_request():
captured_requests: list[httpx.Request] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_extra_body",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={
"fallbacks": [{"model": "claude-opus-5-5"}],
"model": "claude-opus-5-5",
"messages": [{"role": "user", "content": "unchecked"}],
"system": "unchecked",
"max_tokens": 4096,
},
)
finally:
client.close()
request_body = json.loads(captured_requests[0].content)
assert request_body == {
"model": "claude-sonnet-5-5",
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
"max_tokens": 16,
"fallbacks": [{"model": "claude-opus-5-5"}],
}, request_body
def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request:
captured_requests: Final[list[httpx.Request]] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_fallbacks",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={"fallbacks": [{"model": "claude-opus-5-5"}]},
extra_headers=extra_headers,
drop_params=drop_params,
)
finally:
client.close()
return captured_requests[0]
@pytest.mark.parametrize(
("fallback_beta", "drop_params", "fallbacks_kept"),
[
("server-side-fallback-2026-07-01", True, True),
("server-side-fallback-2026-06-01", True, True),
(None, False, True),
(None, True, False),
],
)
def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_a_fallback_beta(
fallback_beta: str | None,
drop_params: bool,
fallbacks_kept: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Both betas unlock ``fallbacks`` per https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
(fetched 2026-11-03)"""
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
monkeypatch.setattr(litellm, "drop_params", False)
request: Final = _anthropic_request_with_fallbacks(
None if fallback_beta is None else {"anthropic-beta": fallback_beta}, drop_params
)
sent_betas: Final = request.headers.get("anthropic-beta", "").split(",")
assert json.loads(request.content).get("fallbacks") == ([{"model": "claude-opus-5-5"}] if fallbacks_kept else None)
assert fallback_beta is None or fallback_beta in sent_betas
def test_anthropic_chat_prices_a_server_side_fallback_answer_by_the_attempt_that_served_it() -> None:
"""Response shape from https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback (fetched
2026-11-03): top-level usage is the attempt that served the turn, usage.iterations records every attempt"""
served_by_fallback: Final = {
"id": "msg_01XFUDYJgAACzvnptvVoYEL",
"type": "message",
"role": "assistant",
"model": "claude-opus-4-8",
"content": [
{
"type": "fallback",
"from": {"model": "claude-fable-5"},
"to": {"model": "claude-opus-4-8"},
"trigger": {"type": "refusal", "category": "cyber"},
},
{"type": "text", "text": "Hi! How can I help you today?"},
],
"stop_reason": "end_turn",
"stop_details": None,
"usage": {
"input_tokens": 412,
"output_tokens": 264,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"iterations": [
{"type": "message", "model": "claude-fable-5", "input_tokens": 535, "output_tokens": 0},
{"type": "fallback_message", "model": "claude-opus-4-8", "input_tokens": 412, "output_tokens": 264},
],
},
}
client: Final = HTTPHandler(
client=httpx.Client(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=served_by_fallback)))
)
try:
response: Final = litellm.completion(
model="anthropic/claude-fable-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=300,
)
finally:
client.close()
prompt_cost, completion_cost = litellm.cost_per_token(
model="claude-opus-4-8", prompt_tokens=412, completion_tokens=264
)
assert response.model == "claude-opus-4-8"
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (412, 264)
assert response._hidden_params["response_cost"] == pytest.approx(prompt_cost + completion_cost)
def test_anthropic_tool_history_without_tools_injects_a_dummy_tool(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:

27
tests/unit/proxy/auth/test_auth_utils.py
View file

@ -3522,6 +3522,33 @@ class TestIsRequestBodySafeBlocksFallbackSmuggle:
)
@pytest.mark.parametrize(
"extra_body",
[
{"fallbacks": "default"},
{"fallbacks": [{"model": "claude-opus-5-5"}]},
'{"fallbacks": [{"model": "claude-opus-5-5"}]}',
],
)
def test_client_chosen_server_side_fallbacks_in_extra_body_are_rejected(extra_body: object) -> None:
with pytest.raises(ValueError, match="fallbacks inside extra_body"):
is_request_body_safe(
request_body={"model": "claude-sonnet-5-5", "extra_body": extra_body},
general_settings={},
llm_router=None,
model="claude-sonnet-5-5",
)
def test_extra_body_without_fallbacks_is_allowed() -> None:
assert is_request_body_safe(
request_body={"model": "claude-sonnet-5-5", "extra_body": {"stop_sequences": ["5"]}},
general_settings={},
llm_router=None,
model="claude-sonnet-5-5",
)
class TestIsRequestBodySafeRejectsUrlValuedFallback:
@pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"])
def test_url_valued_string_fallback_is_rejected(self, fallback_field):