scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-08 08:08:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

Merge 9380f5ea01 into e8f572b39b

Browse source
This commit is contained in:
Mikalai Karytka 2026-10-30 22:50:25 +01:00 • committed by GitHub
commit ef84903294
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 142 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -124,12 +124,14 @@ pub enum AnthropicBeta {
ToolSearchTool20251019,
#[strum(serialize = "inline-tools-2026-09-15")]
InlineTools20260915,
#[strum(serialize = "server-side-fallback-2026-07-01")]
ServerSideFallback20260701,
#[strum(default, transparent)]
Other(String),
}
impl AnthropicBeta {
pub const KNOWN: [Self; 40] = [
pub const KNOWN: [Self; 41] = [
Self::Oauth20250420,
Self::WebFetch20250910,
Self::WebSearch20250305,
@ -170,6 +172,7 @@ impl AnthropicBeta {
Self::ToolExamples20251029,
Self::ToolSearchTool20251019,
Self::InlineTools20260915,
Self::ServerSideFallback20260701,
];
pub fn as_str(&self) -> &str {
@ -248,7 +251,7 @@ impl AnthropicBeta {
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
Self::FastMode20260201 | Self::Oauth20250420 | Self::ServerSideFallback20260701 => &[
BetaProvider::AzureAi,
BetaProvider::BedrockConverse,
BetaProvider::Bedrock,
@ -451,7 +454,8 @@ mod tests {
AnthropicBeta::TokenEfficientTools20250219,
AnthropicBeta::ToolExamples20251029,
AnthropicBeta::ToolSearchTool20251019,
AnthropicBeta::InlineTools20260915
AnthropicBeta::InlineTools20260915,
AnthropicBeta::ServerSideFallback20260701
)]
beta: AnthropicBeta,
) {

14
litellm/anthropic_beta_gated_fields.py
View file

@ -29,6 +29,7 @@ from litellm.types.llms.anthropic import (
ANTHROPIC_MID_CONVERSATION_SYSTEM_CLEAR_AT_BETA_HEADER: Final = "mid-conversation-system-clear-at-2026-08-21"
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: Final = "inline-tools-2026-09-15"
ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: Final = "server-side-fallback-2026-07-01"
THINKING_BLOCK_BINDING: Final = "thinking.block_binding"
THINKING_DISPLAY_UPDATES: Final = "thinking.display=updates"
@ -36,6 +37,7 @@ SYSTEM_CLEAR_AT: Final = "messages[].system.clear_at"
SYSTEM_OUTPUT_CONFIG: Final = "messages[].system.output_config"
SYSTEM_TOOL_CHANGES: Final = "messages[].system.content[].tool_addition|tool_removal"
SYSTEM_MESSAGE_EMPTIED: Final = "messages[].system (emptied by the strip)"
FALLBACKS: Final = "fallbacks"
_TOOL_CHANGE_BLOCK_TYPES: Final = frozenset({"tool_addition", "tool_removal"})
@ -48,6 +50,7 @@ BETA_GATED_FIELDS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value: (SYSTEM_OUTPUT_CONFIG,),
ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_INLINE_TOOLS_BETA_HEADER: (SYSTEM_TOOL_CHANGES,),
ANTHROPIC_SERVER_SIDE_FALLBACK_BETA_HEADER: (FALLBACKS,),
}
)
@ -227,8 +230,8 @@ def strip_beta_gated_system_messages(
def strip_beta_gated_request_fields(
request: Mapping[str, object], active_betas: Set[str], *, drop_params: object
) -> StrippedRequest:
"""Apply both strips to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced."""
"""Apply every strip to an Anthropic ``/v1/messages`` shaped body. Returns the same mapping when
nothing is gated, otherwise a shallow copy with ``thinking`` and ``messages`` replaced and ``fallbacks`` dropped."""
thinking: Final = strip_beta_gated_thinking(request.get("thinking"), active_betas, drop_params=drop_params)
messages: Final = request.get("messages")
message_sequence: Final = _as_sequence(messages) if not isinstance(messages, str) else None
@ -237,11 +240,14 @@ def strip_beta_gated_request_fields(
if message_sequence is not None
else StrippedMessages(messages=(), stripped=())
)
stripped: Final = (*thinking.stripped, *system_messages.stripped)
fallbacks_gated: Final = (
FALLBACKS in request and drops_unsupported_params(drop_params) and field_is_gated(FALLBACKS, active_betas)
)
stripped: Final = (*thinking.stripped, *system_messages.stripped, *((FALLBACKS,) if fallbacks_gated else ()))
if not stripped:
return StrippedRequest(request=request, stripped=())
rewritten: Final = {
**request,
**{key: value for key, value in request.items() if not (fallbacks_gated and key == FALLBACKS)},
**({"thinking": thinking.thinking} if thinking.stripped else {}),
**({"messages": system_messages.messages} if system_messages.stripped else {}),
}

3
litellm/anthropic_beta_headers_config.json
View file

@ -37,7 +37,8 @@
"mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
"inline-tools-2026-09-15": "inline-tools-2026-09-15"
"inline-tools-2026-09-15": "inline-tools-2026-09-15",
"server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01"
},
"azure_ai": {
"advisor-tool-2026-03-01": null,

4
litellm/llms/anthropic/chat/transformation.py
View file

@ -107,6 +107,7 @@ from litellm.types.utils import (
)
from litellm.types.utils import Message as LitellmMessage
from litellm.utils import (
EXTRA_BODY_ROUTING_KEYS,
ModelResponse,
Usage,
add_dummy_tool,
@ -167,6 +168,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_EXTRA_BODY_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2421,6 +2423,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
request_params: Final = cast( # cast-ok: optional_params is the untyped dict of the shared transform signature
dict[str, object], optional_params
)
extra_body: Final = _EXTRA_BODY_ADAPTER.validate_python(request_params.pop("extra_body", None) or {})
marked_tools: Final = tools_marked_for_a_tool_config_point(
request_params.get("tools"), request_params.pop("cache_control_injection_points", None)
)
@ -2458,6 +2461,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"messages": anthropic_messages,
**optional_params,
**({} if marked_tools is None else {"tools": list(marked_tools)}),
**{k: v for k, v in extra_body.items() if k not in EXTRA_BODY_ROUTING_KEYS},
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)

100
tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -136,6 +136,106 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
assert "default_api_key_tpm_limit" not in request_body
def test_anthropic_completion_sends_extra_body_fields_top_level_without_redirecting_model():
captured_requests: list[httpx.Request] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_extra_body",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={"fallbacks": [{"model": "claude-opus-5-5"}], "model": "claude-opus-5-5"},
)
finally:
client.close()
request_body = json.loads(captured_requests[0].content)
assert request_body == {
"model": "claude-sonnet-5-5",
"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
"max_tokens": 16,
"fallbacks": [{"model": "claude-opus-5-5"}],
}, request_body
def _anthropic_request_with_fallbacks(extra_headers: dict[str, str] | None, drop_params: bool) -> httpx.Request:
captured_requests: Final[list[httpx.Request]] = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "msg_fallbacks",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
litellm.completion(
model="anthropic/claude-sonnet-5-5",
messages=[{"role": "user", "content": "Hello"}],
api_key="test-key",
client=client,
max_tokens=16,
extra_body={"fallbacks": "default"},
extra_headers=extra_headers,
drop_params=drop_params,
)
finally:
client.close()
return captured_requests[0]
@pytest.mark.parametrize(
("extra_headers", "drop_params", "expected_fallbacks"),
[
({"anthropic-beta": "server-side-fallback-2026-07-01"}, True, "default"),
(None, False, "default"),
(None, True, None),
],
)
def test_anthropic_chat_drops_fallbacks_under_drop_params_only_when_the_request_lacks_the_fallback_beta(
extra_headers: dict[str, str] | None,
drop_params: bool,
expected_fallbacks: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "true")
monkeypatch.setattr(litellm, "drop_params", False)
request: Final = _anthropic_request_with_fallbacks(extra_headers, drop_params)
assert json.loads(request.content).get("fallbacks") == expected_fallbacks
assert ("server-side-fallback-2026-07-01" in request.headers.get("anthropic-beta", "")) == (
extra_headers is not None
)
def test_anthropic_tool_history_without_tools_injects_a_dummy_tool(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:

19
tests/unit/test_anthropic_beta_headers_filtering.py
View file

@ -509,6 +509,25 @@ class TestAnthropicBetaHeadersFiltering:
assert filtered == ["inline-tools-2026-09-15"]
@pytest.mark.parametrize(
("provider", "expected"),
[
("anthropic", ["server-side-fallback-2026-07-01"]),
("vertex_ai", []),
("bedrock", []),
],
)
def test_server_side_fallback_forwarded_only_to_the_claude_api(self, provider: str, expected: list[str]) -> None:
"""Anthropic rejects the `fallbacks` request field with 400 "fallbacks: Extra inputs are not permitted" unless
this beta is sent, and documents it for the Claude API only
(https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback, fetched 2026-10-26)"""
filtered: Final = filter_and_transform_beta_headers(
beta_headers=["server-side-fallback-2026-07-01"],
provider=provider,
)
assert filtered == expected
def test_null_value_headers_filtered(self):
"""Test that headers with null values are always filtered out."""
for provider in [