This commit is contained in:
Mikalai Karytka 2026-11-03 07:21:48 +00:00 • committed by GitHub
commit 57cd2a695d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 168 additions and 6 deletions

View file

@ -29,6 +29,8 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: Final = "server-side-fallback-2026-06-01"
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: Final = "server-side-fallback-2026-07-01"
THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
@ -36,6 +38,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
FALLBACKS: Final = "fallbacks"
_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
@ -48,6 +51,8 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_06_01_BETA_HEADER: (FALLBACKS,),
ANTHROPIC_SERVER_SIDE_FALLBACK_2026_07_01_BETA_HEADER: (FALLBACKS,),
}
)
@ -227,8 +232,8 @@ def strip_beta_gated_system_messages(
def strip_beta_gated_request_fields(
request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
) -> StrippedRequest:
"""Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
"""Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped."""
thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
messages: Final = request.get("messages")
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
@ -237,11 +242,14 @@ def strip_beta_gated_request_fields(
if message_sequence is not None
else StrippedMessages(messages=(), stripped=())
)
stripped: Final = (*thinking.stripped, *system_messages.stripped)
fallbacks_gated: Final = (
FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas)
)
stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ()))
if not stripped:
return StrippedRequest(request=request, stripped=())
rewritten: Final = {
**request,
**{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)},
**({"thinking": thinking.thinking} if thinking.stripped else {}),
**({"messages": system_messages.messages} if system_messages.stripped else {}),
}

View file

@ -120,6 +120,7 @@ from litellm.types.utils import (
)
from litellm.types.utils import Message as LitellmMessage
from litellm.utils import (
EXTRA_BODY_ROUTING_KEYS,
ModelResponse,
Usage,
add_dummy_tool,
@ -184,6 +185,7 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_EXTRA_BODY_GUARDED_KEYS: Final = EXTRA_BODY_ROUTING_KEYS | frozenset({"messages", "system"})
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2555,6 +2557,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
dict[str, object], optional_params
)
extra_body: Final = _REQUEST_PARAMS_ADAPTER.validate_python(request_params.pop("extra_body", None) or {})
marked_tools: Final = tools_marked_for_a_tool_config_point(
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
)
@ -2587,12 +2590,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
):
request_params.pop("output_format")
data: Final = {
request_body: Final = {
"model": model,
"messages": anthropic_messages,
**optional_params,
**request_params,
**({} if marked_tools is None else {"tools": list(marked_tools)}),
}
withheld_extra_body_keys: Final = request_body.keys() | _EXTRA_BODY_GUARDED_KEYS
data: Final = {
**request_body,
**{k: v for k, v in extra_body.items() if k not in withheld_extra_body_keys},
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)

View file

@ -703,6 +703,7 @@ def is_request_body_safe(
nested = _coerce_metadata_to_dict(request_body.get(nested_key))
if nested is not None:
_check_banned_params(nested, general_settings, llm_router, model, manages_deployments=manages_deployments)
_reject_client_chosen_fallbacks(nested_key, nested)
for metadata_key in _NESTED_METADATA_KEYS:
metadata = _coerce_metadata_to_dict(request_body.get(metadata_key))
if metadata is not None:
@ -738,6 +739,14 @@ def is_request_body_safe(
return True
def _reject_client_chosen_fallbacks(nested_key: str, nested: Mapping[str, object]) -> None:
if nested_key == "extra_body" and "fallbacks" in nested:
raise ValueError(
"Rejected Request: fallbacks inside extra_body can't be checked against the key's allowed models. "
"Set server-side fallbacks in the deployment's litellm_params instead."
)
def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
if manages_deployments:
reject_server_owned_wif_fields(

View file

@ -138,6 +138,116 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
assert "default_api_key_tpm_limit" not in request_body
def test_anthropic_completion_adds_extra_body_fields_top_level_without_overriding_the_request():
captured_requests: list[httpx.Request] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_extra_body",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={
"fallbacks": [{"model": "claude-opus-5-5"}],
"model": "claude-opus-5-5",
"messages": [{"role": "user", "content": "unchecked"}],
"system": "unchecked",
"max_tokens": 4096,
},
)
finally:
client.close()
request_body = json.loads(captured_requests[0].content)
assert request_body == {
"model": "claude-sonnet-5-5",
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
"max_tokens": 16,
"fallbacks": [{"model": "claude-opus-5-5"}],
}, request_body
def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request:
captured_requests: Final[list[httpx.Request]] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_fallbacks",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={"fallbacks": [{"model": "claude-opus-5-5"}]},
extra_headers=extra_headers,
drop_params=drop_params,
)
finally:
client.close()
return captured_requests[0]
@pytest.mark.parametrize(
("fallback_beta", "drop_params", "fallbacks_kept"),
[
("server-side-fallback-2026-07-01", True, True),
("server-side-fallback-2026-06-01", True, True),
(None, False, True),
(None, True, False),
],
)
def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_a_fallback_beta(
fallback_beta: str | None,
drop_params: bool,
fallbacks_kept: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Both betas unlock ``fallbacks`` per https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
(fetched 2026-11-03)"""
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
monkeypatch.setattr(litellm, "drop_params", False)
request: Final = _anthropic_request_with_fallbacks(
None if fallback_beta is None else {"anthropic-beta": fallback_beta}, drop_params
)
sent_betas: Final = request.headers.get("anthropic-beta", "").split(",")
assert json.loads(request.content).get("fallbacks") == ([{"model": "claude-opus-5-5"}] if fallbacks_kept else None)
assert fallback_beta is None or fallback_beta in sent_betas
def test_anthropic_tool_history_without_tools_injects_a_dummy_tool(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:

View file

@ -3522,6 +3522,33 @@ class TestIsRequestBodySafeBlocksFallbackSmuggle:
)
@pytest.mark.parametrize(
"extra_body",
[
{"fallbacks": "default"},
{"fallbacks": [{"model": "claude-opus-5-5"}]},
'{"fallbacks": [{"model": "claude-opus-5-5"}]}',
],
)
def test_client_chosen_server_side_fallbacks_in_extra_body_are_rejected(extra_body: object) -> None:
with pytest.raises(ValueError, match="fallbacks inside extra_body"):
is_request_body_safe(
request_body={"model": "claude-sonnet-5-5", "extra_body": extra_body},
general_settings={},
llm_router=None,
model="claude-sonnet-5-5",
)
def test_extra_body_without_fallbacks_is_allowed() -> None:
assert is_request_body_safe(
request_body={"model": "claude-sonnet-5-5", "extra_body": {"stop_sequences": ["5"]}},
general_settings={},
llm_router=None,
model="claude-sonnet-5-5",
)
class TestIsRequestBodySafeRejectsUrlValuedFallback:
@pytest.mark.parametrize("fallback_field", ["fallbacks", "context_window_fallbacks", "content_policy_fallbacks"])
def test_url_valued_string_fallback_is_rejected(self, fallback_field):