diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 6dcc2ce1976..e284845b2ff 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -49,6 +49,7 @@ _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) _DECODED_JSON: Final = TypeAdapter(object) BetaPlacement: TypeAlias = Literal["header", "body"] +UnlistedBetas: TypeAlias = Literal["drop", "forward"] _HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only" @@ -281,12 +282,13 @@ def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, placement: BetaPlacement = "header", + unlisted: UnlistedBetas = "drop", ) -> list[str]: """ Filter and transform beta headers based on provider's mapping configuration. This function: - 1. Only allows headers that are present in the provider's mapping keys + 1. Only allows headers that are present in the provider's mapping keys, unless ``unlisted`` is "forward" 2. Filters out headers with null values (unsupported) 3. Maps headers to provider-specific names (e.g., advanced-tool-use -> tool-search-tool) @@ -294,6 +296,7 @@ def filter_and_transform_beta_headers( beta_headers: List of Anthropic beta header values provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai") placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field + unlisted: Whether a beta missing from the mapping is dropped or forwarded as sent Returns: List of filtered and transformed beta headers for the provider @@ -308,6 +311,10 @@ def filter_and_transform_beta_headers( for header in beta_headers: header = header.strip() + if header not in provider_mapping and unlisted == "forward": + filtered_headers.add(header) + continue + # Check if header is in the mapping if header not in provider_mapping: verbose_logger.debug( @@ -389,7 +396,11 @@ def get_provider_beta_header( return provider_mapping[anthropic_beta_header] -FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = ("services.ai.azure.com", "openai.azure.com") +FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = ( + "services.ai.azure.com", + "openai.azure.com", + "cognitiveservices.azure.com", +) def is_foundry_api_base(api_base: str | None) -> bool: @@ -416,6 +427,7 @@ def update_headers_with_filtered_beta( headers: dict, provider: str, api_base: str | None = None, + unlisted: UnlistedBetas = "drop", ) -> dict: """ Update headers dict by filtering and transforming anthropic-beta header values. @@ -425,6 +437,7 @@ def update_headers_with_filtered_beta( headers: Request headers dict (will be modified in place) provider: Provider name api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``) + unlisted: Whether a beta missing from the row is dropped or forwarded as sent Returns: Updated headers dict @@ -440,6 +453,7 @@ def update_headers_with_filtered_beta( filtered_beta_values: Final = filter_and_transform_beta_headers( beta_headers=beta_values, provider=resolve_beta_provider(provider, api_base), + unlisted=unlisted, ) # Update or remove the header diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index a0435147d37..ecfd0371eb3 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -3168,8 +3168,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): STOP_REASON_FIELDS.validate_python(completion_response) ) completion_response["stop_reason"] = "stop" - if stop_reason_fields: - json_mode_message.provider_specific_fields = dict(stop_reason_fields) + input_transformations: Final = provider_specific_fields.get("input_transformations") + json_mode_fields: Final = { + **stop_reason_fields, + **({} if input_transformations is None else {"input_transformations": input_transformations}), + } + if json_mode_fields: + json_mode_message.provider_specific_fields = json_mode_fields if "refusal" in refusal_fields: json_mode_message.refusal = refusal_fields["refusal"] if thinking_blocks: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index abac8132117..cd88fab5e64 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2591,9 +2591,8 @@ class BaseLLMHTTPHandler: takes_another_beta_row: Final = ( resolve_beta_provider(custom_llm_provider, affinity_api_base) != custom_llm_provider ) - filters_beta_headers: Final = ( - anthropic_messages_provider_config.should_filter_anthropic_beta_headers() or takes_another_beta_row - ) + provider_filters_betas: Final = anthropic_messages_provider_config.should_filter_anthropic_beta_headers() + filters_beta_headers: Final = provider_filters_betas or takes_another_beta_row if request_params.get("safeguards") is not None and filters_beta_headers: await aload_beta_headers_config() default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) @@ -2614,9 +2613,13 @@ class BaseLLMHTTPHandler: api_base=api_base, ) - if filters_beta_headers: + if filters_beta_headers and headers.get("anthropic-beta") is not None: + await aload_beta_headers_config() headers = update_headers_with_filtered_beta( - headers=headers, provider=custom_llm_provider, api_base=affinity_api_base + headers=headers, + provider=custom_llm_provider, + api_base=affinity_api_base, + unlisted="drop" if provider_filters_betas else "forward", ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 35136c41dc2..de67e1b123e 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -119,6 +119,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( { **_FEATURES, Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, + Feature.FOUNDRY_BETA_ROW: route_detectors.foundry_beta_row, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry, diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 629d9098970..9ba8cbc35a7 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -66,6 +66,8 @@ class Feature(str, Enum): """Extra body fields that Python transforms and merges into the provider request.""" PROVIDER_AFFINITY = "provider_affinity" """A provider affinity header that Python validates and populates from the session id.""" + FOUNDRY_BETA_ROW = "foundry_beta_row" + """A Foundry api_base, where Python drops the anthropic-beta values Foundry rejects.""" PRE_REQUEST_HOOK = "pre_request_hook" """A logger overriding async_pre_request_hook, which may rewrite the request.""" WEBSEARCH_INTERCEPTION = "websearch_interception" @@ -170,6 +172,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { Feature.PROVIDER_AFFINITY, + Feature.FOUNDRY_BETA_ROW, Feature.SAFEGUARDS_CLASSIFIER, Feature.PRE_REQUEST_HOOK, Feature.WEBSEARCH_INTERCEPTION, diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 8edd2707405..6a0c867979b 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Final import litellm +from litellm.anthropic_beta_headers_manager import is_foundry_api_base from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, overridden, present, streaming @@ -15,6 +16,12 @@ def provider_affinity(request: NativeCall) -> bool: return is_opencode_api_base(api_base(request)) +def foundry_beta_row(request: NativeCall) -> bool: + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + return is_foundry_api_base(AnthropicModelInfo.get_api_base(api_base(request))) + + def anthropic_workload_identity(request: NativeCall) -> bool: """Python mints an Anthropic federation token when no static credential is configured.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 5b11e628b8d..6d65f289524 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -569,6 +569,12 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict( ASYNC_ONLY_FEATURES: Final = ( pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), + pytest.param( + Feature.FOUNDRY_BETA_ROW, + {"api_base": "https://team.services.ai.azure.com/anthropic"}, + {}, + id="foundry-beta-row", + ), pytest.param( Feature.WEBSEARCH_INTERCEPTION, {}, {"callbacks": [WebSearchInterceptionLogger()]}, id="websearch-interception" ), diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 0586538f34b..7577562d5ed 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -40,6 +40,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im VertexAIAnthropicConfig, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -10017,18 +10018,18 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): _INPUT_TRANSFORMATIONS: Final = [{"type": "thinking_block_binding", "count": 2}] -def _input_transformations_response() -> dict: - return { - "id": "msg_input_transformations", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-4-6", - "content": [{"type": "text", "text": "ok"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": 12, "output_tokens": 3}, - "input_transformations": _INPUT_TRANSFORMATIONS, - } +def _input_transformations_response() -> AnthropicMessagesResponse: + return AnthropicMessagesResponse( + id="msg_input_transformations", + type="message", + role="assistant", + model="claude-sonnet-4-6", + content=[{"type": "text", "text": "ok"}], + stop_reason="end_turn", + stop_sequence=None, + usage={"input_tokens": 12, "output_tokens": 3}, + input_transformations=_INPUT_TRANSFORMATIONS, + ) def test_chat_response_surfaces_input_transformations_in_provider_specific_fields(): @@ -10048,8 +10049,9 @@ def test_chat_response_surfaces_input_transformations_in_provider_specific_field def test_chat_response_without_input_transformations_adds_no_key(): - completion_response = _input_transformations_response() - del completion_response["input_transformations"] + completion_response: Final = { + key: value for key, value in _input_transformations_response().items() if key != "input_transformations" + } result = AnthropicConfig().transform_response( model="claude-sonnet-4-6", raw_response=httpx.Response(200, json=completion_response), @@ -10065,6 +10067,29 @@ def test_chat_response_without_input_transformations_adds_no_key(): assert "input_transformations" not in result.choices[0].message.provider_specific_fields +def test_json_mode_chat_response_keeps_input_transformations(): + completion_response: Final = { + **_input_transformations_response(), + "content": [{"type": "tool_use", "id": "toolu_json", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {"answer": 42}}], + "stop_reason": "tool_use", + } + result: Final = AnthropicConfig().transform_response( + model="claude-sonnet-4-6", + raw_response=httpx.Response(200, json=completion_response), + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + json_mode=True, + ) + + assert json.loads(result.choices[0].message.content) == {"answer": 42} + assert result.choices[0].message.provider_specific_fields["input_transformations"] == _INPUT_TRANSFORMATIONS + + _KEPT_SEARCH_CALL: Final = { "role": "assistant", "content": None, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 8190fed86d7..97e2bfee4b0 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -6265,29 +6265,59 @@ async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( client.client.close() +_BETA_THE_TABLE_DOES_NOT_LIST: Final = "not-in-the-beta-table-2099-01-01" + + @pytest.mark.asyncio @pytest.mark.parametrize( - ("custom_llm_provider", "pass_through", "forwarded_betas"), + ("custom_llm_provider", "messages_config", "foundry_api_base", "forwarded_betas"), [ - pytest.param("anthropic", False, "thinking-binding-controls-2026-08-01", id="anthropic-takes-the-azure-ai-row"), + pytest.param( + "anthropic", + "anthropic", + "https://my-resource.services.ai.azure.com/anthropic", + f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}", + id="anthropic-drops-what-the-azure-ai-row-rejects", + ), + pytest.param( + "anthropic", + "anthropic", + "https://my-resource.cognitiveservices.azure.com/anthropic", + f"thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}", + id="anthropic-on-a-cognitiveservices-host", + ), + pytest.param( + "azure_ai", + "azure_ai", + "https://my-resource.services.ai.azure.com/anthropic", + "thinking-binding-controls-2026-08-01", + id="azure-ai-keeps-dropping-unlisted-betas", + ), pytest.param( "openai", - True, - "advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01", + "pass_through", + "https://my-resource.services.ai.azure.com/anthropic", + f"advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}", id="messages-pass-through-forwards-as-sent", ), ], ) async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment_on_a_foundry_api_base( - monkeypatch, custom_llm_provider: str, pass_through: bool, forwarded_betas: str + monkeypatch, custom_llm_provider: str, messages_config: str, foundry_api_base: str, forwarded_betas: str ): from litellm import anthropic_beta_headers_manager from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") anthropic_beta_headers_manager._BETA_HEADERS_CONFIG = None - foundry_api_base: Final = "https://my-resource.services.ai.azure.com/anthropic" + assert _BETA_THE_TABLE_DOES_NOT_LIST not in anthropic_beta_headers_manager._load_beta_headers_config()["azure_ai"] + configs: Final = { + "anthropic": AnthropicMessagesConfig, + "azure_ai": AzureAnthropicMessagesConfig, + "pass_through": OpenAILikeAnthropicMessagesConfig, + } upstream_response: Final = httpx.Response( 200, json={ @@ -6311,9 +6341,7 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment await BaseLLMHTTPHandler().async_anthropic_messages_handler( model="claude-sonnet-4-6", messages=[{"role": "user", "content": "Hello"}], - anthropic_messages_provider_config=( - OpenAILikeAnthropicMessagesConfig() if pass_through else AnthropicMessagesConfig() - ), + anthropic_messages_provider_config=configs[messages_config](), anthropic_messages_optional_request_params={"max_tokens": 8}, custom_llm_provider=custom_llm_provider, litellm_params=GenericLiteLLMParams(api_base=foundry_api_base), @@ -6321,11 +6349,80 @@ async def test_async_anthropic_messages_handler_picks_the_beta_row_by_deployment api_key="sk-test", api_base=foundry_api_base, client=mock_client, - kwargs={"extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01"}}, + kwargs={ + "extra_headers": { + "anthropic-beta": ( + f"advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01,{_BETA_THE_TABLE_DOES_NOT_LIST}" + ) + } + }, ) sent_headers: Final = mock_client.post.call_args.kwargs["headers"] - assert sent_headers["anthropic-beta"] == forwarded_betas + assert set(sent_headers["anthropic-beta"].split(",")) == set(forwarded_betas.split(",")) + + +@pytest.mark.asyncio +async def test_first_foundry_request_with_betas_fetches_the_beta_table_off_the_event_loop(monkeypatch): + from litellm import anthropic_beta_headers_manager, anthropic_beta_headers_url + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) + monkeypatch.setattr(anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + bundled_table: Final = anthropic_beta_headers_manager.GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() + fetch_started: Final = threading.Event() + loop_answered: Final = threading.Event() + + def table_held_until_the_loop_answers(request: httpx.Request) -> httpx.Response: + fetch_started.set() + return httpx.Response(200 if loop_answered.wait(timeout=2) else 504, json=bundled_table) + + foundry_api_base: Final = "https://my-resource.services.ai.azure.com/anthropic" + mock_client: Final = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello!"}], + "model": "claude-sonnet-4-6", + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=httpx.Request("POST", f"{foundry_api_base}/v1/messages"), + ) + ) + mock_logging_obj: Final = Mock() + mock_logging_obj.model_call_details = {} + mock_logging_obj.stream = False + mock_logging_obj.dynamic_success_callbacks = None + + with respx.mock(assert_all_called=True) as router: + table_route: Final = router.get(anthropic_beta_headers_url).mock(side_effect=table_held_until_the_loop_answers) + request: Final = asyncio.create_task( + BaseLLMHTTPHandler().async_anthropic_messages_handler( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_provider_config=AnthropicMessagesConfig(), + anthropic_messages_optional_request_params={"max_tokens": 8}, + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_base=foundry_api_base), + logging_obj=mock_logging_obj, + api_key="sk-test", + api_base=foundry_api_base, + client=mock_client, + kwargs={"extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,compact-2026-01-12"}}, + ) + ) + assert await asyncio.to_thread(fetch_started.wait, 5) + loop_answered.set() + await request + + assert table_route.calls.last.response.status_code == 200 + assert mock_client.post.call_args.kwargs["headers"]["anthropic-beta"] == "compact-2026-01-12" @pytest.mark.asyncio diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 634e69910b9..e4c39bb40f9 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -334,6 +334,17 @@ def test_stream_cost_injection_needs_a_stream(monkeypatch: pytest.MonkeyPatch) - assert not messages_detectors.stream_cost_injection(call()) +def test_foundry_beta_row_reads_the_api_base_the_anthropic_handler_reads(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + assert messages_detectors.foundry_beta_row(call(api_base="https://team.services.ai.azure.com/anthropic")) + assert messages_detectors.foundry_beta_row(call(base_url="https://team.cognitiveservices.azure.com/anthropic")) + assert not messages_detectors.foundry_beta_row(call(api_base="https://api.anthropic.com")) + assert not messages_detectors.foundry_beta_row(call()) + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://team.services.ai.azure.com/anthropic") + assert messages_detectors.foundry_beta_row(call()) + + def test_model_alias_map_applies_only_to_the_requested_model(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_alias_map", {"fast": "openai/gpt-5-mini"}) assert chat_detectors.model_alias_map(call(model="fast"))