diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index d68de6c9d4d..0fbc2b67e4b 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -31,6 +31,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": "web-fetch-2025-09-10", "web-search-2025-03-05": "web-search-2025-03-05", @@ -68,6 +69,9 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", + "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": null, "web-fetch-2025-09-10": "web-fetch-2025-09-10", @@ -102,6 +106,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21": null, "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": null, "tool-search-tool-2025-10-19": null, @@ -137,6 +142,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21": null, "token-efficient-tools-2025-02-19": null, "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", "web-fetch-2025-09-10": null, @@ -175,6 +181,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21": null, "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "tool-examples-2025-10-29": "tool-examples-2025-10-29", @@ -211,6 +218,9 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", + "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": null, "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", @@ -248,6 +258,7 @@ "text_editor_20241022": null, "text_editor_20250124": null, "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21": null, "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": "web-search-2025-03-05" diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index fec1d8e4465..3c9f7c2bc70 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -31,6 +31,7 @@ from concurrent.futures import ThreadPoolExecutor from functools import cache from importlib.resources import files from typing import Final, TypeAlias +from urllib.parse import urlparse import httpx from pydantic import TypeAdapter @@ -355,9 +356,33 @@ def get_provider_beta_header( return provider_mapping[anthropic_beta_header] +FOUNDRY_API_BASE_HOST_SUFFIXES: Final[tuple[str, ...]] = ("services.ai.azure.com", "openai.azure.com") + + +def is_foundry_api_base(api_base: str | None) -> bool: + """Whether ``api_base`` points at an Azure AI Foundry host, where Claude is served by Microsoft.""" + if not api_base: + return False + host: Final = urlparse(api_base if "://" in api_base else f"https://{api_base}").hostname or "" + return any(host == suffix or host.endswith(f".{suffix}") for suffix in FOUNDRY_API_BASE_HOST_SUFFIXES) + + +def resolve_beta_provider(provider: str, api_base: str | None = None) -> str: + """ + Pick the beta allowlist row for one deployment, not one provider string. + + An ``anthropic`` deployment whose ``api_base`` is a Foundry host is served by Microsoft and + gets the ``azure_ai`` row, since Foundry accepts a different set of betas than api.anthropic.com. + """ + if get_provider_name(provider) == "anthropic" and is_foundry_api_base(api_base): + return "azure_ai" + return provider + + def update_headers_with_filtered_beta( headers: dict, provider: str, + api_base: str | None = None, ) -> dict: """ Update headers dict by filtering and transforming anthropic-beta header values. @@ -366,6 +391,7 @@ def update_headers_with_filtered_beta( Args: headers: Request headers dict (will be modified in place) provider: Provider name + api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``) Returns: Updated headers dict @@ -380,7 +406,7 @@ def update_headers_with_filtered_beta( # Filter and transform based on provider filtered_beta_values: Final = filter_and_transform_beta_headers( beta_headers=beta_values, - provider=provider, + provider=resolve_beta_provider(provider, api_base), ) # Update or remove the header @@ -397,6 +423,7 @@ def update_request_with_filtered_beta( headers: dict, request_data: dict, provider: str, + api_base: str | None = None, ) -> tuple[dict, dict]: """ Update both headers and request body beta fields based on provider support. @@ -406,11 +433,13 @@ def update_request_with_filtered_beta( headers: Request headers dict (will be modified in place) request_data: Request body dict (will be modified in place) provider: Provider name + api_base: Deployment api_base, used to pick the allowlist row (Foundry hosts use ``azure_ai``) Returns: Tuple of (updated headers, updated request_data) """ - headers = update_headers_with_filtered_beta(headers=headers, provider=provider) + row_provider: Final = resolve_beta_provider(provider, api_base) + headers = update_headers_with_filtered_beta(headers=headers, provider=row_provider) existing_body_betas: Final = request_data.get("anthropic_beta") if not existing_body_betas: @@ -418,7 +447,7 @@ def update_request_with_filtered_beta( filtered_body_betas: Final = filter_and_transform_beta_headers( beta_headers=existing_body_betas, - provider=provider, + provider=row_provider, ) if filtered_body_betas: diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 13ceea3e31b..d9931f80fb1 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -509,6 +509,7 @@ class AnthropicChatCompletion(BaseLLM): headers=headers, request_data=request_data, provider=custom_llm_provider, + api_base=api_base, ) logging_obj.pre_call( input=messages, @@ -1165,6 +1166,9 @@ class ModelResponseIterator: start_message: Final = message_start_block["message"] if "model" in start_message: self.served_model = start_message["model"] + input_transformations: Final = start_message.get("input_transformations") + if input_transformations is not None: + provider_specific_fields["input_transformations"] = input_transformations if "usage" in message_start_block["message"]: usage = self._handle_usage(anthropic_usage_chunk=message_start_block["message"]["usage"]) elif type_chunk == "error": diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..1410758a4ad 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2937,6 +2937,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if context_management is not None: provider_specific_fields["context_management"] = context_management + input_transformations: Final = completion_response.get("input_transformations") + if input_transformations is not None: + provider_specific_fields["input_transformations"] = input_transformations + if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results provider_specific_fields["web_search_calls"] = self._build_web_search_calls( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ef009e70137..6b7a50b1bd5 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -36,7 +36,11 @@ import litellm.litellm_core_utils import litellm.types import litellm.types.utils from litellm._logging import redact_string, verbose_logger -from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, update_headers_with_filtered_beta +from litellm.anthropic_beta_headers_manager import ( + aload_beta_headers_config, + is_foundry_api_base, + update_headers_with_filtered_beta, +) from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs @@ -2494,14 +2498,15 @@ class BaseLLMHTTPHandler: litellm_params.get("additional_drop_params") or (), anthropic_messages_optional_request_params, ) - if ( - request_params.get("safeguards") is not None - and anthropic_messages_provider_config.should_filter_anthropic_beta_headers() - ): - await aload_beta_headers_config() affinity_api_base: Final = ( AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base ) + filters_beta_headers: Final = ( + anthropic_messages_provider_config.should_filter_anthropic_beta_headers() + or is_foundry_api_base(affinity_api_base) + ) + if request_params.get("safeguards") is not None and filters_beta_headers: + await aload_beta_headers_config() default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) ( headers, @@ -2520,8 +2525,10 @@ class BaseLLMHTTPHandler: api_base=api_base, ) - if anthropic_messages_provider_config.should_filter_anthropic_beta_headers(): - headers = update_headers_with_filtered_beta(headers=headers, provider=custom_llm_provider) + if filters_beta_headers: + headers = update_headers_with_filtered_beta( + headers=headers, provider=custom_llm_provider, api_base=affinity_api_base + ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index ba422a0a901..33174ff22c1 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -651,6 +651,7 @@ class MessageChunk(TypedDict, total=False): stop_sequence: str | None usage: UsageDelta safeguard_results: ReadOnly[Sequence[SafeguardResult]] + input_transformations: ReadOnly[Sequence[Mapping[str, object]]] class MessageStartBlock(TypedDict): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 9ad13243451..56c012436f4 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,4 +1,4 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -106,3 +106,4 @@ class AnthropicMessagesResponse(TypedDict, total=False): usage: AnthropicUsage | None context_management: NotRequired[ContextManagementResponse] safeguard_results: NotRequired[ReadOnly[Sequence[SafeguardResult]]] + input_transformations: NotRequired[ReadOnly[Sequence[Mapping[str, object]]]] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 1f4a5aaa256..efea2ad7dd0 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -494,7 +494,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou {"type": "content_block_delta", "index": 2, "delta": {"type": "thinking_delta", "thinking": "Step 3."}}, {"type": "content_block_delta", "index": 2, "delta": {"type": "signature_delta", "signature": "sig-b"}}, {"type": "content_block_stop", "index": 2}, - {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 9}}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 9}, + }, {"type": "message_stop"}, ) @@ -515,7 +519,11 @@ def test_streamed_thinking_blocks_accumulate_in_the_openai_sdk_and_replay_withou "thinking_blocks": message["thinking_blocks"], } replayed: Final = anthropic_messages_pt( - messages=[{"role": "user", "content": "Capital of France?"}, next_turn, {"role": "user", "content": "And Spain?"}], + messages=[ + {"role": "user", "content": "Capital of France?"}, + next_turn, + {"role": "user", "content": "And Spain?"}, + ], model="claude-haiku-4-5", llm_provider="anthropic", ) @@ -3607,3 +3615,37 @@ def test_streamed_end_turn_carries_no_stop_details(): provider_fields: Final = assembled.choices[0].message.provider_specific_fields or {} assert "stop_details" not in provider_fields assert "stop_reason" not in provider_fields + + +def test_message_start_surfaces_input_transformations_on_the_first_chunk(): + input_transformations: Final = [{"type": "thinking_block_binding", "count": 2}] + chunk: Final = { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 1}, + "input_transformations": input_transformations, + }, + } + iterator: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False) + + first_chunk: Final = iterator.chunk_parser(chunk=chunk) + + assert first_chunk.choices[0].delta.provider_specific_fields == {"input_transformations": input_transformations} + assert first_chunk.model == "claude-sonnet-4-6" + + +def test_message_start_without_input_transformations_has_no_provider_specific_fields(): + chunk: Final = { + "type": "message_start", + "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-sonnet-4-6", "content": []}, + } + iterator: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False) + + assert iterator.chunk_parser(chunk=chunk).choices[0].delta.provider_specific_fields is None diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index ef002a73e52..a95b2a94bbf 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -9804,3 +9804,54 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +_INPUT_TRANSFORMATIONS: Final = [{"type": "thinking_block_binding", "count": 2}] + + +def _input_transformations_response() -> dict: + return { + "id": "msg_input_transformations", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3}, + "input_transformations": _INPUT_TRANSFORMATIONS, + } + + +def test_chat_response_surfaces_input_transformations_in_provider_specific_fields(): + result = AnthropicConfig().transform_response( + model="claude-sonnet-4-6", + raw_response=httpx.Response(200, json=_input_transformations_response()), + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert result.choices[0].message.provider_specific_fields["input_transformations"] == _INPUT_TRANSFORMATIONS + + +def test_chat_response_without_input_transformations_adds_no_key(): + completion_response = _input_transformations_response() + del completion_response["input_transformations"] + result = AnthropicConfig().transform_response( + model="claude-sonnet-4-6", + raw_response=httpx.Response(200, json=completion_response), + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "input_transformations" not in result.choices[0].message.provider_specific_fields diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index aa6625561ba..23422e45000 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -416,3 +416,33 @@ def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_re assert request["messages"][1] == 7 assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}] + + +def test_native_response_keeps_input_transformations() -> None: + input_transformations: Final[list[JsonValue]] = [{"type": "thinking_block_binding", "count": 2}] + upstream: Final[dict[str, JsonValue]] = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 19, "output_tokens": 42}, + "input_transformations": input_transformations, + } + logging_obj: Final = Logging( + model="claude-sonnet-4-6", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-call", + function_id="test-function", + ) + + response: Final = AnthropicMessagesConfig().transform_anthropic_messages_response( + model="claude-sonnet-4-6", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj + ) + + assert response["input_transformations"] == input_transformations diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 297e64629a8..cb3f1edc929 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -413,9 +413,7 @@ def test_response_api_handler_sends_no_stream_options_when_web_search_forces_a_n ): sent: Final = _SentRequestRecorder() recorder: Final = _AgenticLoopKwargsRecorder() - monkeypatch.setattr( - litellm, "callbacks", [WebSearchInterceptionLogger(enabled_providers=["azure"]), recorder] - ) + monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger(enabled_providers=["azure"]), recorder]) logging_obj: Final = Mock() logging_obj.dynamic_success_callbacks = [] @@ -2486,7 +2484,9 @@ async def test_async_anthropic_messages_handler_passes_deployment_api_base_to_ag super().__init__() self.hook_kwargs: dict | None = None - async def async_should_run_agentic_loop(self, response, model, messages, tools, stream, custom_llm_provider, kwargs): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): self.hook_kwargs = dict(kwargs) return False, {} @@ -2846,9 +2846,7 @@ def _transform_subtitle_response(payload): def test_subtitle_synthesis_fallback_without_timings_drops_words(): - response = _transform_subtitle_response( - {"text": "hello world", "words": [{"word": "hello"}, {"word": "world"}]} - ) + response = _transform_subtitle_response({"text": "hello world", "words": [{"word": "hello"}, {"word": "world"}]}) assert response.text == "hello world" assert "words" not in response @@ -3840,7 +3838,13 @@ def _capture_video_create_request(captured): captured["body"] = request.content return httpx.Response( 200, - json={"id": "video_123", "object": "video", "status": "queued", "created_at": 1712697600, "model": "sora-2"}, + json={ + "id": "video_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + "model": "sora-2", + }, ) return respond @@ -3863,7 +3867,9 @@ def test_video_generation_without_file_sends_multipart_form_data(): captured = {} client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) - result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(OpenAIVideoConfig())) + result = BaseLLMHTTPHandler().video_generation_handler( + client=client, **_video_create_call_kwargs(OpenAIVideoConfig()) + ) assert captured["content_type"].startswith("multipart/form-data") assert _multipart_text_fields(captured["content_type"], captured["body"]) == { @@ -3903,7 +3909,9 @@ def test_azure_video_generation_without_file_sends_multipart_form_data(): captured = {} client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) - result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(AzureVideoConfig())) + result = BaseLLMHTTPHandler().video_generation_handler( + client=client, **_video_create_call_kwargs(AzureVideoConfig()) + ) assert captured["content_type"].startswith("multipart/form-data") assert _multipart_text_fields(captured["content_type"], captured["body"]) == { @@ -3918,7 +3926,9 @@ def test_video_generation_json_provider_keeps_json_body(): captured = {} client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_capture_video_create_request(captured)))) - result = BaseLLMHTTPHandler().video_generation_handler(client=client, **_video_create_call_kwargs(_JSONBodyVideoConfig())) + result = BaseLLMHTTPHandler().video_generation_handler( + client=client, **_video_create_call_kwargs(_JSONBodyVideoConfig()) + ) assert captured["content_type"] == "application/json" assert json.loads(captured["body"]) == {"model": "sora-2", "prompt": "a cat surfing", "seconds": "4"} @@ -3948,6 +3958,7 @@ def test_video_generation_with_input_reference_keeps_file_multipart(): AZURE_AI_BASE = "https://myfoundry.services.ai.azure.com" AZURE_AI_CHAT_COMPLETIONS_URL = f"{AZURE_AI_BASE}/models/chat/completions" + def _a_tool_with_an_unsupported_field() -> dict: return { "type": "function", @@ -3955,14 +3966,13 @@ def _a_tool_with_an_unsupported_field() -> dict: "strict": True, } + A_COMPLETION = { "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "grok-3", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "sent"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } @@ -4006,9 +4016,7 @@ def _call_azure_ai(recorder: _RecordedAzureAI, **overrides): def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried(): - recorder = _RecordedAzureAI( - [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] - ) + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]) response = _call_azure_ai(recorder) @@ -4019,9 +4027,7 @@ def test_a_tool_field_the_provider_rejects_is_dropped_and_the_call_retried(): def test_the_retry_changes_only_the_field_the_provider_named(): - recorder = _RecordedAzureAI( - [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] - ) + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]) _call_azure_ai(recorder) @@ -4060,9 +4066,7 @@ def test_an_extra_input_outside_a_tool_is_not_retried_unless_dropping_params_was def test_an_extra_input_outside_a_tool_is_retried_when_dropping_params_was_asked_for(): - recorder = _RecordedAzureAI( - [_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)] - ) + recorder = _RecordedAzureAI([_rejection(UNRELATED_REJECTION), httpx.Response(200, json=A_COMPLETION)]) response = _call_azure_ai(recorder, drop_params=True) @@ -4076,9 +4080,7 @@ async def test_a_tool_field_the_provider_rejects_is_dropped_and_retried_on_the_a ): import respx - recorder = _RecordedAzureAI( - [_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)] - ) + recorder = _RecordedAzureAI([_rejection(TOOL_LEVEL_REJECTION), httpx.Response(200, json=A_COMPLETION)]) with respx.mock(assert_all_called=True) as router: router.post(AZURE_AI_CHAT_COMPLETIONS_URL).mock(side_effect=recorder) @@ -4219,7 +4221,11 @@ async def test_create_batch_says_which_setting_is_missing_when_the_provider_reso with pytest.raises(ValueError, match="api_base is required for create_batch"): await BaseLLMHTTPHandler().create_batch( - create_batch_data={"input_file_id": "file_1", "endpoint": "/v1/chat/completions", "completion_window": "24h"}, + create_batch_data={ + "input_file_id": "file_1", + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, litellm_params={}, provider_config=provider_config, headers={}, @@ -4400,7 +4406,9 @@ def _start_async_completion(config, logging_obj=None): custom_llm_provider="openai", model_response=ModelResponse(), encoding=None, - logging_obj=logging_obj if logging_obj is not None else Mock(dynamic_success_callbacks=None, model_call_details={}), + logging_obj=logging_obj + if logging_obj is not None + else Mock(dynamic_success_callbacks=None, model_call_details={}), optional_params={}, timeout=10.0, litellm_params={}, @@ -4788,7 +4796,9 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid logging_obj.dispatch_failure_handlers = AsyncMock() handler = BaseLLMHTTPHandler() - with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(side_effect=AssertionError("dialed a websocket"))) as dial: + with patch.object( + handler, "_open_realtime_backend_ws", AsyncMock(side_effect=AssertionError("dialed a websocket")) + ) as dial: await handler.async_realtime( model="chirp_3", websocket=client_ws, @@ -5599,7 +5609,9 @@ async def test_messages_opencode_session_header( model="anthropic/claude-haiku-5-5", messages=[{"role": "user", "content": "hello"}], max_tokens=128000, - api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base, + api_base=None + if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" + else api_base, api_key="test-key", stream=stream, litellm_session_id="conversation-123", @@ -5660,17 +5672,27 @@ def test_vector_store_retrieve_handler_sends_id_before_azure_api_version_query() @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.parametrize("provider", ("openai", "azure")) async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( - is_async: bool, provider: str, respx_mock: respx.MockRouter, + is_async: bool, + provider: str, + respx_mock: respx.MockRouter, ) -> None: route: Final = respx_mock.post(url__regex=r"https://responses\.example\.com/.*").respond( 200, json={ - "id": "resp_headers", "object": "response", "created_at": 1, - "model": "gpt-6-luna", "status": "completed", "output": [], - "parallel_tool_calls": False, "tool_choice": "auto", "tools": [], + "id": "resp_headers", + "object": "response", + "created_at": 1, + "model": "gpt-6-luna", + "status": "completed", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], }, ) - client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) if is_async else HTTPHandler(client=httpx.Client()) + client: Final = ( + AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) if is_async else HTTPHandler(client=httpx.Client()) + ) kwargs: Final = { "model": f"{provider}/gpt-6-luna", "input": "hello", @@ -5695,3 +5717,49 @@ async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( await client.close() else: client.client.close() + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_filters_betas_by_the_azure_ai_row_on_a_foundry_api_base(monkeypatch): + from litellm import anthropic_beta_headers_manager + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig + + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + anthropic_beta_headers_manager._BETA_HEADERS_CONFIG = None + foundry_api_base = "https://my-resource.services.ai.azure.com/anthropic" + upstream_response = httpx.Response( + 200, + json={ + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello!"}], + "model": "claude-sonnet-4-6", + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=httpx.Request("POST", f"{foundry_api_base}/v1/messages"), + ) + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=upstream_response) + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + mock_logging_obj.stream = False + mock_logging_obj.dynamic_success_callbacks = None + + await BaseLLMHTTPHandler().async_anthropic_messages_handler( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_provider_config=AnthropicMessagesConfig(), + anthropic_messages_optional_request_params={"max_tokens": 8}, + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(api_base=foundry_api_base), + logging_obj=mock_logging_obj, + api_key="sk-test", + api_base=foundry_api_base, + client=mock_client, + kwargs={"extra_headers": {"anthropic-beta": "advisor-tool-2026-03-01,thinking-binding-controls-2026-08-01"}}, + ) + + sent_headers = mock_client.post.call_args.kwargs["headers"] + assert sent_headers["anthropic-beta"] == "thinking-binding-controls-2026-08-01" diff --git a/tests/unit/test_anthropic_beta_headers_deployment_rows.py b/tests/unit/test_anthropic_beta_headers_deployment_rows.py new file mode 100644 index 00000000000..777b01e4e53 --- /dev/null +++ b/tests/unit/test_anthropic_beta_headers_deployment_rows.py @@ -0,0 +1,108 @@ +"""The beta allowlist row is picked per deployment: an ``anthropic`` deployment on a Foundry host gets ``azure_ai``.""" + +import json +import os +from typing import Final + +import pytest + +import litellm +from litellm import anthropic_beta_headers_manager +from litellm.anthropic_beta_headers_manager import ( + is_foundry_api_base, + resolve_beta_provider, + update_headers_with_filtered_beta, + update_request_with_filtered_beta, +) + +FOUNDRY_API_BASE: Final = "https://my-resource.services.ai.azure.com/anthropic" +DIRECT_API_BASE: Final = "https://api.anthropic.com" +DIRECT_ONLY_BETA: Final = "advisor-tool-2026-03-01" +PRESERVED_THINKING_BETAS: Final = ( + "thinking-binding-controls-2026-08-01", + "mid-conversation-system-clear-at-2026-08-21", + "mid-conversation-tool-changes-2026-07-01", + "mid-conversation-output-config-2026-07-01", + "thinking-display-updates-2026-08-18", +) + + +@pytest.fixture(autouse=True) +def _bundled_config(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + anthropic_beta_headers_manager._BETA_HEADERS_CONFIG = None + + +def _bundled_rows() -> dict: + with open(os.path.join(os.path.dirname(litellm.__file__), "anthropic_beta_headers_config.json")) as f: + return json.load(f) + + +@pytest.mark.parametrize( + ("api_base", "expected"), + ( + (FOUNDRY_API_BASE, True), + ("https://my-resource.openai.azure.com", True), + ("my-resource.services.ai.azure.com/anthropic", True), + (DIRECT_API_BASE, False), + ("https://services.ai.azure.com.example.com", False), + ("", False), + (None, False), + ), +) +def test_is_foundry_api_base_matches_only_foundry_hosts(api_base: str | None, expected: bool) -> None: + assert is_foundry_api_base(api_base) is expected + + +@pytest.mark.parametrize( + ("provider", "api_base", "expected"), + ( + ("anthropic", FOUNDRY_API_BASE, "azure_ai"), + ("anthropic", DIRECT_API_BASE, "anthropic"), + ("anthropic", None, "anthropic"), + ("vertex_ai", FOUNDRY_API_BASE, "vertex_ai"), + ("bedrock_converse", None, "bedrock_converse"), + ), +) +def test_resolve_beta_provider_picks_the_row_per_deployment(provider: str, api_base: str | None, expected: str) -> None: + assert resolve_beta_provider(provider, api_base) == expected + + +def test_foundry_deployment_headers_follow_the_azure_ai_row() -> None: + headers: Final = {"anthropic-beta": f"{DIRECT_ONLY_BETA},thinking-binding-controls-2026-08-01"} + result: Final = update_headers_with_filtered_beta(headers=headers, provider="anthropic", api_base=FOUNDRY_API_BASE) + assert result == {"anthropic-beta": "thinking-binding-controls-2026-08-01"} + + +def test_direct_deployment_headers_keep_every_anthropic_row_beta() -> None: + headers: Final = {"anthropic-beta": f"{DIRECT_ONLY_BETA},thinking-binding-controls-2026-08-01"} + result: Final = update_headers_with_filtered_beta(headers=headers, provider="anthropic", api_base=DIRECT_API_BASE) + assert result == {"anthropic-beta": f"{DIRECT_ONLY_BETA},thinking-binding-controls-2026-08-01"} + + +def test_foundry_deployment_request_filters_header_and_body_betas_together() -> None: + headers, data = update_request_with_filtered_beta( + headers={"anthropic-beta": DIRECT_ONLY_BETA}, + request_data={"anthropic_beta": [DIRECT_ONLY_BETA, "thinking-binding-controls-2026-08-01"], "max_tokens": 8}, + provider="anthropic", + api_base=FOUNDRY_API_BASE, + ) + assert headers == {} + assert data == {"anthropic_beta": ["thinking-binding-controls-2026-08-01"], "max_tokens": 8} + + +@pytest.mark.parametrize("provider", ("anthropic", "azure_ai", "vertex_ai")) +@pytest.mark.parametrize("beta", PRESERVED_THINKING_BETAS) +def test_live_verified_platforms_forward_every_preserved_thinking_beta(provider: str, beta: str) -> None: + assert _bundled_rows()[provider][beta] == beta + + +@pytest.mark.parametrize("provider", ("bedrock", "bedrock_converse", "bedrock_mantle", "databricks")) +def test_unverified_platforms_hold_back_the_system_clear_beta(provider: str) -> None: + assert _bundled_rows()[provider]["mid-conversation-system-clear-at-2026-08-21"] is None + assert ( + update_headers_with_filtered_beta( + headers={"anthropic-beta": "mid-conversation-system-clear-at-2026-08-21"}, provider=provider + ) + == {} + )