From 3497184b4ebf308d2338c452901d6279374f84f0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:19:20 -0700 Subject: [PATCH 1/7] fix(anthropic): stream azure_ai natively on an Azure OpenAI host on /v1/messages The /v1/messages bridges got the requested model (azure_ai/gpt-6-luna) with the resolved provider (azure), so the Responses bridge faked the stream and the chat bridge called a deployment named azure_ai/gpt-6-luna (404). Pass the resolved provider prefix when the requested prefix is a different known provider. --- .../llms/anthropic/messages/native/handler.py | 13 ++++-- .../native/test_anthropic_messages_handler.py | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..a088ce93055 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -44,7 +44,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import CallTypes, LlmProviders +from litellm.types.utils import CallTypes, LlmProviders, LlmProvidersSet from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator @@ -94,6 +94,13 @@ def _responses_mode_is_lost_by_prefix_strip( ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) +def _bridge_model(requested_model: str, resolved_model: str, custom_llm_provider: str) -> str: + requested_provider: Final = requested_model.partition("/")[0] + if requested_provider == custom_llm_provider or requested_provider not in LlmProvidersSet: + return requested_model + return f"{custom_llm_provider}/{resolved_model}" + + def _should_route_to_responses_api( custom_llm_provider: str | None, requested_model: str | None = None, @@ -698,7 +705,7 @@ def anthropic_messages_handler( return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, - model=original_model, + model=_bridge_model(original_model, model, custom_llm_provider), metadata=metadata, stop_sequences=stop_sequences, stream=stream, @@ -728,7 +735,7 @@ def anthropic_messages_handler( return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, - model=original_model, + model=_bridge_model(original_model, model, custom_llm_provider), metadata=metadata, stop_sequences=stop_sequences, stream=stream, diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index c11f58faec4..d2787d3912a 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -6,6 +6,7 @@ import uuid from typing import Any, Dict, Final, List import httpx +import openai import responses import respx import pytest @@ -1131,6 +1132,49 @@ async def test_messages_strips_provider_prefix_exactly_once(requested_model, exp assert captured["url"] == expected_url +_GET_WEATHER_TOOL: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tools, expected_path", + [ + ([_GET_WEATHER_TOOL], "/openai/v1/responses"), + (None, "/openai/deployments/gpt-6-luna/chat/completions"), + ], + ids=["function_tool_bridged_to_responses", "no_tools_on_chat"], +) +async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_deployment(tools, expected_path): + captured = {} + + async def fake_send(self, request, **kwargs): + captured["body"] = json.loads(request.content) + captured["url"] = request.url + raise httpx.ConnectError("cut at the wire", request=request) + + with ( + patch.object(httpx.AsyncClient, "send", fake_send), + pytest.raises(openai.APIError), + ): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "What is the weather in Berlin?"}], + model="azure_ai/gpt-6-luna", + api_base="https://qa-resource.cognitiveservices.azure.com", + api_key="test-api-key", + stream=True, + tools=tools, + ) + + assert captured["url"].path == expected_path + assert captured["body"]["model"] == "gpt-6-luna" + assert captured["body"]["stream"] is True + + @pytest.mark.asyncio @pytest.mark.parametrize( "model_info, expected_url", From 027cf1eaa40b474108d8e5b9ddb272b4f3d6fd13 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:32:50 -0700 Subject: [PATCH 2/7] test(anthropic): cover the bridged model with respx and a regional bedrock_mantle id --- .../native/test_anthropic_messages_handler.py | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index d2787d3912a..2807992054c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1148,31 +1148,47 @@ _GET_WEATHER_TOOL: Final = { ], ids=["function_tool_bridged_to_responses", "no_tools_on_chat"], ) -async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_deployment(tools, expected_path): - captured = {} +async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_deployment(tools, expected_path, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() - async def fake_send(self, request, **kwargs): - captured["body"] = json.loads(request.content) - captured["url"] = request.url - raise httpx.ConnectError("cut at the wire", request=request) - - with ( - patch.object(httpx.AsyncClient, "send", fake_send), - pytest.raises(openai.APIError), - ): - await litellm.anthropic.messages.acreate( - max_tokens=100, - messages=[{"role": "user", "content": "What is the weather in Berlin?"}], - model="azure_ai/gpt-6-luna", - api_base="https://qa-resource.cognitiveservices.azure.com", - api_key="test-api-key", - stream=True, - tools=tools, + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route(host="qa-resource.cognitiveservices.azure.com").mock( + side_effect=httpx.ConnectError("cut at the wire") ) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "What is the weather in Berlin?"}], + model="azure_ai/gpt-6-luna", + api_base="https://qa-resource.cognitiveservices.azure.com", + api_key="test-api-key", + stream=True, + tools=tools, + ) - assert captured["url"].path == expected_path - assert captured["body"]["model"] == "gpt-6-luna" - assert captured["body"]["stream"] is True + bodies: Final = [json.loads(call.request.content) for call in upstream.calls] + assert {call.request.url.path for call in upstream.calls} == {expected_path} + assert {body["model"] for body in bodies} == {"gpt-6-luna"} + assert all(body["stream"] is True for body in bodies) + + +@pytest.mark.asyncio +async def test_bridged_messages_keep_the_region_of_a_regional_bedrock_mantle_model(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(litellm.exceptions.ServiceUnavailableError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="bedrock_mantle/eu-west-1/openai.gpt-oss-120b", + api_key="test-api-key", + ) + + assert {call.request.url.host for call in upstream.calls} == {"bedrock-mantle.eu-west-1.api.aws"} @pytest.mark.asyncio From c72bb54dc2c3ba91b9804d15ba4e844168390cac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:02:54 -0700 Subject: [PATCH 3/7] fix(anthropic): carry azure_ai env credentials into bridged /v1/messages calls When an azure_ai deployment's api_base and key come only from AZURE_AI_API_BASE and AZURE_AI_API_KEY, the provider the async wrapper derived from the model alone replaced the model's own resolution, so the bridged call lost both and failed. The model's own resolution now wins when it names the same provider, an explicit provider that disagrees still wins, and the bridge carries the resolved key and base. --- .../llms/anthropic/messages/native/handler.py | 62 ++++++++++++++----- .../native/test_anthropic_messages_handler.py | 39 +++++++++++- 2 files changed, 81 insertions(+), 20 deletions(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index a088ce93055..21bbd72fdc1 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -8,6 +8,7 @@ import asyncio import contextvars from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import partial from typing import Final, cast @@ -94,11 +95,39 @@ def _responses_mode_is_lost_by_prefix_strip( ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) -def _bridge_model(requested_model: str, resolved_model: str, custom_llm_provider: str) -> str: - requested_provider: Final = requested_model.partition("/")[0] +@dataclass(frozen=True, slots=True) +class _BridgeTarget: + model: str + api_key: str | None + api_base: str | None + + +def _resolve_requested_provider( + model: str, custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams +) -> tuple[str, str, str | None, str | None]: + resolved: Final = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + litellm_params=litellm_params, + ) + requested_provider: Final = model.partition("/")[0] + if custom_llm_provider in (None, requested_provider) or requested_provider not in LlmProvidersSet: + return resolved + own: Final = litellm.get_llm_provider(model=model, api_base=litellm_params.api_base, api_key=litellm_params.api_key) + return own if own[1] == custom_llm_provider else resolved + + +def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm_provider: str) -> _BridgeTarget: + requested_provider: Final = requested.model.partition("/")[0] if requested_provider == custom_llm_provider or requested_provider not in LlmProvidersSet: - return requested_model - return f"{custom_llm_provider}/{resolved_model}" + return requested + return _BridgeTarget( + model=f"{custom_llm_provider}/{resolved.model}", + api_key=resolved.api_key, + api_base=resolved.api_base, + ) def _should_route_to_responses_api( @@ -629,13 +658,7 @@ def anthropic_messages_handler( custom_llm_provider, dynamic_api_key, dynamic_api_base, - ) = litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=litellm_params.api_base, - api_key=litellm_params.api_key, - litellm_params=litellm_params, - ) + ) = _resolve_requested_provider(model, custom_llm_provider, litellm_params) # Store agentic loop params in logging object for agentic hooks # This provides original request context needed for follow-up calls @@ -698,6 +721,11 @@ def anthropic_messages_handler( model, custom_llm_provider, kwargs.get("model_info"), api_base ) if anthropic_messages_provider_config is None: + bridge: Final = _bridge_target( + _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base), + _BridgeTarget(model=model, api_key=dynamic_api_key, api_base=dynamic_api_base), + custom_llm_provider, + ) # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences @@ -705,7 +733,7 @@ def anthropic_messages_handler( return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, - model=_bridge_model(original_model, model, custom_llm_provider), + model=bridge.model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, @@ -721,8 +749,8 @@ def anthropic_messages_handler( top_k=top_k, top_p=top_p, _is_async=is_async, - api_key=api_key, - api_base=api_base, + api_key=bridge.api_key, + api_base=bridge.api_base, client=client, custom_llm_provider=custom_llm_provider, **kwargs, @@ -735,7 +763,7 @@ def anthropic_messages_handler( return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, - model=_bridge_model(original_model, model, custom_llm_provider), + model=bridge.model, metadata=metadata, stop_sequences=stop_sequences, stream=stream, @@ -749,8 +777,8 @@ def anthropic_messages_handler( top_k=top_k, top_p=top_p, _is_async=is_async, - api_key=api_key, - api_base=api_base, + api_key=bridge.api_key, + api_base=bridge.api_base, client=client, custom_llm_provider=custom_llm_provider, **kwargs, diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 2807992054c..5707bafb1c3 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1148,9 +1148,21 @@ _GET_WEATHER_TOOL: Final = { ], ids=["function_tool_bridged_to_responses", "no_tools_on_chat"], ) -async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_deployment(tools, expected_path, monkeypatch): +@pytest.mark.parametrize("credentials_from_env", [False, True], ids=["credentials_in_params", "credentials_from_env"]) +async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_deployment( + tools, expected_path, credentials_from_env, monkeypatch +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.delenv("AZURE_API_BASE", raising=False) + monkeypatch.delenv("AZURE_API_KEY", raising=False) + monkeypatch.setenv("AZURE_AI_API_BASE", "https://qa-resource.cognitiveservices.azure.com") + monkeypatch.setenv("AZURE_AI_API_KEY", "env-api-key") + credentials: Final = ( + {} + if credentials_from_env + else {"api_base": "https://qa-resource.cognitiveservices.azure.com", "api_key": "params-api-key"} + ) with respx.mock(assert_all_called=True) as respx_mock: upstream: Final = respx_mock.route(host="qa-resource.cognitiveservices.azure.com").mock( @@ -1161,14 +1173,14 @@ async def test_azure_ai_on_azure_openai_host_streams_natively_with_the_bare_depl max_tokens=100, messages=[{"role": "user", "content": "What is the weather in Berlin?"}], model="azure_ai/gpt-6-luna", - api_base="https://qa-resource.cognitiveservices.azure.com", - api_key="test-api-key", stream=True, tools=tools, + **credentials, ) bodies: Final = [json.loads(call.request.content) for call in upstream.calls] assert {call.request.url.path for call in upstream.calls} == {expected_path} + assert {call.request.headers["api-key"] for call in upstream.calls} == {credentials.get("api_key", "env-api-key")} assert {body["model"] for body in bodies} == {"gpt-6-luna"} assert all(body["stream"] is True for body in bodies) @@ -1191,6 +1203,27 @@ async def test_bridged_messages_keep_the_region_of_a_regional_bedrock_mantle_mod assert {call.request.url.host for call in upstream.calls} == {"bedrock-mantle.eu-west-1.api.aws"} +@pytest.mark.asyncio +async def test_bridged_messages_keep_the_callers_explicit_provider_over_the_model_prefix(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="anthropic/claude-sonnet-5-5", + custom_llm_provider="openai", + api_base="https://gateway.example.com/v1", + api_key="test-api-key", + ) + + assert {call.request.url.host for call in upstream.calls} == {"gateway.example.com"} + assert {json.loads(call.request.content)["model"] for call in upstream.calls} == {"anthropic/claude-sonnet-5-5"} + + @pytest.mark.asyncio @pytest.mark.parametrize( "model_info, expected_url", From 97e5917ab4ead8f6770836863f69402418c5031e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:32:04 -0700 Subject: [PATCH 4/7] fix(anthropic): keep the caller's key, proxy routing and MCP credentials on bridged /v1/messages calls --- .../llms/anthropic/messages/native/handler.py | 35 +++++++++---- .../native/test_anthropic_messages_handler.py | 49 +++++++++++++++++-- .../messages/native/test_mcp_handler.py | 32 ++++++++++++ 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 21bbd72fdc1..b677ca94331 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -115,8 +115,22 @@ def _resolve_requested_provider( requested_provider: Final = model.partition("/")[0] if custom_llm_provider in (None, requested_provider) or requested_provider not in LlmProvidersSet: return resolved - own: Final = litellm.get_llm_provider(model=model, api_base=litellm_params.api_base, api_key=litellm_params.api_key) - return own if own[1] == custom_llm_provider else resolved + own: Final = _resolve_by_model_prefix(model, litellm_params) + return own if own is not None and own[1] == custom_llm_provider else resolved + + +def _resolve_by_model_prefix( + model: str, litellm_params: GenericLiteLLMParams +) -> tuple[str, str, str | None, str | None] | None: + try: + return litellm.get_llm_provider( + model=model, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + litellm_params=litellm_params.model_copy(update={"custom_llm_provider": None}), + ) + except litellm.BadRequestError: + return None def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm_provider: str) -> _BridgeTarget: @@ -125,7 +139,7 @@ def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm return requested return _BridgeTarget( model=f"{custom_llm_provider}/{resolved.model}", - api_key=resolved.api_key, + api_key=requested.api_key or resolved.api_key, api_base=resolved.api_base, ) @@ -684,6 +698,12 @@ def anthropic_messages_handler( mock_response=litellm_params.mock_response, ) + bridge: Final = _bridge_target( + _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base), + _BridgeTarget(model=model, api_key=dynamic_api_key, api_base=dynamic_api_base), + custom_llm_provider, + ) + # Expand litellm_proxy MCP references through the MCP gateway before dispatch, so every # downstream path (native passthrough and both bridges) gets real tools rather than a # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider. @@ -710,8 +730,8 @@ def anthropic_messages_handler( top_k=top_k, top_p=top_p, container=container, - api_key=api_key, - api_base=api_base, + api_key=bridge.api_key, + api_base=bridge.api_base, client=client, custom_llm_provider=custom_llm_provider, **kwargs, @@ -721,11 +741,6 @@ def anthropic_messages_handler( model, custom_llm_provider, kwargs.get("model_info"), api_base ) if anthropic_messages_provider_config is None: - bridge: Final = _bridge_target( - _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base), - _BridgeTarget(model=model, api_key=dynamic_api_key, api_base=dynamic_api_base), - custom_llm_provider, - ) # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 5707bafb1c3..1c240186d3c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1204,7 +1204,8 @@ async def test_bridged_messages_keep_the_region_of_a_regional_bedrock_mantle_mod @pytest.mark.asyncio -async def test_bridged_messages_keep_the_callers_explicit_provider_over_the_model_prefix(monkeypatch): +@pytest.mark.parametrize("model", ["anthropic/claude-sonnet-5-5", "ragflow/gpt-4o"]) +async def test_bridged_messages_keep_the_callers_explicit_provider_over_the_model_prefix(model, monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() @@ -1214,14 +1215,56 @@ async def test_bridged_messages_keep_the_callers_explicit_provider_over_the_mode await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "ping"}], - model="anthropic/claude-sonnet-5-5", + model=model, custom_llm_provider="openai", api_base="https://gateway.example.com/v1", api_key="test-api-key", ) assert {call.request.url.host for call in upstream.calls} == {"gateway.example.com"} - assert {json.loads(call.request.content)["model"] for call in upstream.calls} == {"anthropic/claude-sonnet-5-5"} + assert {json.loads(call.request.content)["model"] for call in upstream.calls} == {model} + + +@pytest.mark.asyncio +async def test_bridged_messages_send_the_deployments_key_for_azure_models_served_as_openai(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "server-openai-key") + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="azure/mistral-large-latest", + api_base="https://example-resource.services.ai.azure.com/models", + api_key="deployment-key", + ) + + assert {call.request.headers["authorization"] for call in upstream.calls} == {"Bearer deployment-key"} + + +@pytest.mark.asyncio +async def test_bridged_messages_still_route_through_the_litellm_proxy_when_asked(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LITELLM_PROXY_API_BASE", "https://proxy.example.com") + monkeypatch.setenv("LITELLM_PROXY_API_KEY", "proxy-key") + monkeypatch.setenv("AZURE_AI_API_BASE", "https://example-resource.cognitiveservices.azure.com") + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-env-key") + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="azure_ai/gpt-6-luna", + use_litellm_proxy=True, + ) + + assert {call.request.url.host for call in upstream.calls} == {"proxy.example.com"} @pytest.mark.asyncio diff --git a/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py index 73c2edbce19..82181c8d543 100644 --- a/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py @@ -2,8 +2,12 @@ from importlib import import_module from typing import Final from unittest.mock import AsyncMock, patch +import httpx +import openai import pytest +import respx +import litellm from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -56,6 +60,34 @@ def test_anthropic_messages_handler_routes_litellm_proxy_mcp_to_the_gateway(): assert result is not None +@pytest.mark.asyncio +async def test_anthropic_messages_with_mcp_reaches_an_env_only_deployment_with_its_env_credentials(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("AZURE_AI_API_BASE", "https://example-resource.cognitiveservices.azure.com") + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-env-key") + litellm.in_memory_llm_clients_cache.flush_cache() + + with ( + patch.object( + import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, + "process_mcp_tools_without_openai_transform", + new=AsyncMock(return_value=([], {})), + ), + respx.mock(assert_all_called=True) as respx_mock, + ): + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="azure_ai/gpt-6-luna", + tools=[MCP_REFERENCE], + ) + + assert {call.request.url.host for call in upstream.calls} == {"example-resource.cognitiveservices.azure.com"} + assert {call.request.headers.get("api-key") for call in upstream.calls} == {"azure-env-key"} + + def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): """The gateway's own follow-up call must not re-enter the gateway.""" with patch( From ea5f02e8dd31eea03a2c12cfb5b3f8aa1226bf6d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:21:37 -0700 Subject: [PATCH 5/7] fix(anthropic): keep an azure deployment's Entra credential over the env key on bridged /v1/messages calls --- .../llms/anthropic/messages/native/handler.py | 15 +++++++- .../native/test_anthropic_messages_handler.py | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index b677ca94331..3f495853655 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -133,6 +133,15 @@ def _resolve_by_model_prefix( return None +def _sets_own_entra_credential(litellm_params: GenericLiteLLMParams) -> bool: + return bool( + litellm_params.azure_ad_token + or (litellm_params.model_extra or {}).get("azure_ad_token_provider") is not None + or (litellm_params.tenant_id and litellm_params.client_id and litellm_params.client_secret) + or (litellm_params.azure_username and litellm_params.azure_password and litellm_params.client_id) + ) + + def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm_provider: str) -> _BridgeTarget: requested_provider: Final = requested.model.partition("/")[0] if requested_provider == custom_llm_provider or requested_provider not in LlmProvidersSet: @@ -700,7 +709,11 @@ def anthropic_messages_handler( bridge: Final = _bridge_target( _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base), - _BridgeTarget(model=model, api_key=dynamic_api_key, api_base=dynamic_api_base), + _BridgeTarget( + model=model, + api_key=None if _sets_own_entra_credential(litellm_params) else dynamic_api_key, + api_base=dynamic_api_base, + ), custom_llm_provider, ) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 1c240186d3c..6e7600861ee 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1245,6 +1245,44 @@ async def test_bridged_messages_send_the_deployments_key_for_azure_models_served assert {call.request.headers["authorization"] for call in upstream.calls} == {"Bearer deployment-key"} +@pytest.mark.asyncio +@pytest.mark.parametrize( + "entra_credential", + [ + {"azure_ad_token": "entra-token"}, + {"azure_ad_token_provider": lambda: "entra-token"}, + {"tenant_id": "tenant", "client_id": "client", "client_secret": "secret"}, + {"azure_username": "user", "azure_password": "password", "client_id": "client"}, + ], +) +async def test_bridged_messages_keep_the_deployments_entra_credential_over_the_env_key(monkeypatch, entra_credential): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("AZURE_AI_API_KEY", "azure-env-key") + litellm.in_memory_llm_clients_cache.flush_cache() + + with ( + patch("litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id", return_value=lambda: "entra-token"), # test-quality-ok: stands in for the Entra token fetch so the test can see which credential reaches Azure + patch( + "litellm.llms.azure.common_utils.get_azure_ad_token_from_username_password", + return_value=lambda: "entra-token", + ), # test-quality-ok: stands in for the Entra token fetch so the test can see which credential reaches Azure + respx.mock(assert_all_called=True) as respx_mock, + ): + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="azure_ai/gpt-6-luna", + api_base="https://example-resource.cognitiveservices.azure.com", + tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + **entra_credential, + ) + + assert {call.request.headers.get("authorization") for call in upstream.calls} == {"Bearer entra-token"} + assert {call.request.headers.get("api-key") for call in upstream.calls} == {None} + + @pytest.mark.asyncio async def test_bridged_messages_still_route_through_the_litellm_proxy_when_asked(monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) From cb77c44025d4fbc7674375d623461af73a15191b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:28:25 -0700 Subject: [PATCH 6/7] fix(anthropic): skip the sign-in lookup for chatgpt and github_copilot model prefixes on bridged /v1/messages calls --- .../llms/anthropic/messages/native/handler.py | 7 +++++- .../native/test_anthropic_messages_handler.py | 24 +++++++++++++++++++ .../messages/native/test_mcp_handler.py | 12 +++------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 3f495853655..51b28c9ead8 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -16,6 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm.anthropic_beta_headers_manager import aload_beta_headers_config +from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type @@ -113,7 +114,11 @@ def _resolve_requested_provider( litellm_params=litellm_params, ) requested_provider: Final = model.partition("/")[0] - if custom_llm_provider in (None, requested_provider) or requested_provider not in LlmProvidersSet: + if ( + custom_llm_provider in (None, requested_provider) + or requested_provider not in LlmProvidersSet + or requested_provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO + ): return resolved own: Final = _resolve_by_model_prefix(model, litellm_params) return own if own is not None and own[1] == custom_llm_provider else resolved diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 6e7600861ee..eabc18e2550 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1225,6 +1225,30 @@ async def test_bridged_messages_keep_the_callers_explicit_provider_over_the_mode assert {json.loads(call.request.content)["model"] for call in upstream.calls} == {model} +@pytest.mark.parametrize("model", ["chatgpt/gpt-5", "github_copilot/gpt-5"]) +def test_bridged_messages_reach_the_callers_provider_without_signing_in_to_the_prefix_provider( + model, monkeypatch, tmp_path +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "server-openai-key") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + litellm.anthropic.messages.create( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model=model, + custom_llm_provider="openai", + ) + + assert {call.request.url.host for call in upstream.calls} == {"api.openai.com"} + assert {call.request.headers["authorization"] for call in upstream.calls} == {"Bearer server-openai-key"} + + @pytest.mark.asyncio async def test_bridged_messages_send_the_deployments_key_for_azure_models_served_as_openai(monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py index 82181c8d543..5fe6bb122e8 100644 --- a/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py @@ -67,25 +67,19 @@ async def test_anthropic_messages_with_mcp_reaches_an_env_only_deployment_with_i monkeypatch.setenv("AZURE_AI_API_KEY", "azure-env-key") litellm.in_memory_llm_clients_cache.flush_cache() - with ( - patch.object( - import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, - "process_mcp_tools_without_openai_transform", - new=AsyncMock(return_value=([], {})), - ), - respx.mock(assert_all_called=True) as respx_mock, - ): + with respx.mock(assert_all_called=True) as respx_mock: upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) with pytest.raises(openai.APIError): await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "ping"}], model="azure_ai/gpt-6-luna", - tools=[MCP_REFERENCE], + tools=[{**MCP_REFERENCE, "server_url": "litellm_proxy"}], ) assert {call.request.url.host for call in upstream.calls} == {"example-resource.cognitiveservices.azure.com"} assert {call.request.headers.get("api-key") for call in upstream.calls} == {"azure-env-key"} + assert not any(b"litellm_proxy" in call.request.content for call in upstream.calls) def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): From ae1d8aa7f75bd7c970ee4d23cda598ee1217ab46 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:58:25 -0700 Subject: [PATCH 7/7] fix(anthropic): keep the Azure credential of an azure_ai deployment that sets its own api_base on bridged /v1/messages calls --- .../llms/anthropic/messages/native/handler.py | 2 +- .../native/test_anthropic_messages_handler.py | 50 ++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 51b28c9ead8..27ac0501160 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -153,7 +153,7 @@ def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm return requested return _BridgeTarget( model=f"{custom_llm_provider}/{resolved.model}", - api_key=requested.api_key or resolved.api_key, + api_key=requested.api_key if requested.api_base else requested.api_key or resolved.api_key, api_base=resolved.api_base, ) diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index eabc18e2550..4033105a6be 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -1279,10 +1279,15 @@ async def test_bridged_messages_send_the_deployments_key_for_azure_models_served {"azure_username": "user", "azure_password": "password", "client_id": "client"}, ], ) -async def test_bridged_messages_keep_the_deployments_entra_credential_over_the_env_key(monkeypatch, entra_credential): +@pytest.mark.parametrize("api_base_from_env", [False, True], ids=["api_base_in_params", "api_base_from_env"]) +async def test_bridged_messages_keep_the_deployments_entra_credential_over_the_env_key( + monkeypatch, entra_credential, api_base_from_env +): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("AZURE_AI_API_BASE", "https://example-resource.cognitiveservices.azure.com") monkeypatch.setenv("AZURE_AI_API_KEY", "azure-env-key") litellm.in_memory_llm_clients_cache.flush_cache() + api_base: Final = {} if api_base_from_env else {"api_base": "https://example-resource.cognitiveservices.azure.com"} with ( patch("litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id", return_value=lambda: "entra-token"), # test-quality-ok: stands in for the Entra token fetch so the test can see which credential reaches Azure @@ -1298,8 +1303,8 @@ async def test_bridged_messages_keep_the_deployments_entra_credential_over_the_e max_tokens=100, messages=[{"role": "user", "content": "ping"}], model="azure_ai/gpt-6-luna", - api_base="https://example-resource.cognitiveservices.azure.com", tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + **api_base, **entra_credential, ) @@ -1307,6 +1312,47 @@ async def test_bridged_messages_keep_the_deployments_entra_credential_over_the_e assert {call.request.headers.get("api-key") for call in upstream.calls} == {None} +@pytest.mark.asyncio +@pytest.mark.parametrize( + "azure_env_credential, expected_header, expected_value", + [ + ({"AZURE_API_KEY": "azure-env-key"}, "api-key", "azure-env-key"), + ( + {"AZURE_TENANT_ID": "tenant", "AZURE_CLIENT_ID": "client", "AZURE_CLIENT_SECRET": "secret"}, + "authorization", + "Bearer entra-token", + ), + ], + ids=["azure_env_key", "azure_env_entra"], +) +async def test_bridged_messages_keep_the_azure_env_credential_when_the_deployment_sets_its_own_api_base( + azure_env_credential, expected_header, expected_value, monkeypatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + for name in ("AZURE_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_AD_TOKEN", "AZURE_TENANT_ID", "AZURE_CLIENT_ID"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("AZURE_AI_API_KEY", "key-for-another-resource") + for name, value in azure_env_credential.items(): + monkeypatch.setenv(name, value) + litellm.in_memory_llm_clients_cache.flush_cache() + + with ( + patch("litellm.llms.azure.common_utils.get_azure_ad_token_from_entra_id", return_value=lambda: "entra-token"), # test-quality-ok: stands in for the Entra token fetch so the test can see which credential reaches Azure + respx.mock(assert_all_called=True) as respx_mock, + ): + upstream: Final = respx_mock.route().mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(openai.APIError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="azure_ai/gpt-6-luna", + api_base="https://example-resource.cognitiveservices.azure.com", + tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + ) + + assert {call.request.headers.get(expected_header) for call in upstream.calls} == {expected_value} + + @pytest.mark.asyncio async def test_bridged_messages_still_route_through_the_litellm_proxy_when_asked(monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)