diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ef009e70137..afa12b58b29 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6,6 +6,7 @@ import ssl from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache, partial, reduce +from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -62,7 +63,7 @@ from litellm.litellm_core_utils.realtime_errors import ( ) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -572,6 +573,22 @@ def _unwrap_provider_error(error: Exception) -> Exception: return error +def _merge_messages_header_sources(*sources: Mapping[str, object] | None) -> Mapping[str, object]: + """Later sources win per header, except ``anthropic-beta``, which unions every source so client betas + never drop a deployment's configured ones.""" + entries: Final = tuple(chain.from_iterable((source or {}).items() for source in sources)) + beta_values: Final = tuple( + cast("str | Sequence[str]", value) # cast-ok: the merge helper skips non-str list entries + for name, value in entries + if name.lower() == "anthropic-beta" and isinstance(value, str | Sequence) + ) + merged_beta: Final = reduce(merge_anthropic_beta_headers, beta_values, "") + return MappingProxyType( + {name: value for name, value in entries if name.lower() != "anthropic-beta"} + | ({"anthropic-beta": merged_beta} if merged_beta else {}) + ) + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -2481,13 +2498,8 @@ class BaseLLMHTTPHandler: forwarded_headers: Final = kwargs.get("headers", None) # Also check for extra_headers in kwargs (from config or direct calls) extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None) - # Merge all header sources: forwarded < extra_headers < provider_specific - merged_headers: Final[Mapping[str, object]] = MappingProxyType( - { - **(forwarded_headers or {}), - **(extra_headers_from_kwargs or {}), - **(provider_specific_headers or {}), - } + merged_headers: Final = _merge_messages_header_sources( + forwarded_headers, extra_headers_from_kwargs, provider_specific_headers ) request_params: Final = reduce( delete_nested_value, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 297e64629a8..f10f5a67b2f 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -66,7 +66,13 @@ from litellm.responses.streaming_iterator import ( ) from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, TranscriptionResponse +from litellm.types.utils import ( + ImageObject, + ImageResponse, + ModelResponse, + ProviderSpecificHeader, + TranscriptionResponse, +) from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -1607,6 +1613,89 @@ async def test_async_anthropic_messages_handler_header_priority(): assert captured_headers["X-Provider-Only"] == "keep-this-too" +class _AnthropicMessagesUpstream: + def __init__(self) -> None: + self.anthropic_beta: tuple[str, ...] = () + + def respond(self, request: httpx.Request) -> httpx.Response: + self.anthropic_beta = tuple(request.headers.get_list("anthropic-beta")) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + +_BOTH_FOUNDRY_BETAS: Final = ("context-1m-2025-08-07,context-management-2025-06-27",) + + +@pytest.mark.parametrize( + ("forwarded_headers", "deployment_headers", "client_beta", "expected"), + [ + pytest.param( + None, + {"anthropic-beta": "context-1m-2025-08-07"}, + "context-management-2025-06-27", + _BOTH_FOUNDRY_BETAS, + id="deployment-and-client-betas-are-unioned", + ), + pytest.param( + None, + {"anthropic-beta": ["context-1m-2025-08-07"]}, + "context-management-2025-06-27", + _BOTH_FOUNDRY_BETAS, + id="deployment-list-and-client-string-are-unioned", + ), + pytest.param( + {"anthropic-beta": "interleaved-thinking-2025-05-14"}, + {"anthropic-beta": "context-1m-2025-08-07"}, + "context-management-2025-06-27", + ("context-1m-2025-08-07,context-management-2025-06-27,interleaved-thinking-2025-05-14",), + id="forwarded-deployment-and-client-betas-are-unioned", + ), + pytest.param( + None, + {"anthropic-beta": "context-1m-2025-08-07"}, + "claude-code-20250219", + ("context-1m-2025-08-07",), + id="unsupported-client-beta-is-dropped-and-deployment-beta-kept", + ), + ], +) +@pytest.mark.asyncio +async def test_messages_sends_deployment_and_client_anthropic_betas_upstream( + forwarded_headers: dict[str, str] | None, + deployment_headers: dict[str, str | list[str]], + client_beta: str, + expected: tuple[str, ...], +) -> None: + upstream: Final = _AnthropicMessagesUpstream() + + await litellm.anthropic.messages.acreate( + model="azure_ai/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=16, + api_base="https://test.services.ai.azure.com/anthropic", + api_key="test-key", + headers=forwarded_headers, + extra_headers=deployment_headers, + provider_specific_header=ProviderSpecificHeader( + custom_llm_provider="azure_ai", extra_headers={"anthropic-beta": client_beta} + ), + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream.respond)), + ) + + assert upstream.anthropic_beta == expected + + @pytest.mark.asyncio async def test_async_anthropic_messages_handler_drops_top_level_and_nested_params(): """