From 572f26b0cde09d9152c222ddaee2b1a38df89754 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:28:51 -0700 Subject: [PATCH] fix(proxy): keep the served model when a fallback lands on a raw-name or Azure Model Router group --- litellm/proxy/common_request_processing.py | 40 +++++++++++-- .../test_responses_id_ownership.py | 6 +- .../test_responses_lifecycle_wire.py | 6 +- .../test_fallback_retry_header_contracts.py | 4 +- .../routing/test_observed_routing.py | 2 +- .../proxy/test_common_request_processing.py | 57 +++++++++++++++++++ 6 files changed, 102 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 950ce5ba699..b2b845a8770 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1522,13 +1522,41 @@ def _non_empty_str(value: object) -> str | None: _NO_HEADERS: Final[Mapping[str, object]] = MappingProxyType({}) +def raw_name_router_groups(llm_router: Router | None) -> frozenset[str]: + if llm_router is None: + return frozenset() + return frozenset( + model_group + for model_group, tagged_routers in llm_router.complexity_routers.items() + if any(tagged.strategy.config.return_raw_model_name for tagged in tagged_routers) + ) + + +def _fallback_group_reports_served_model( + model_group: str | None, hidden_params: Mapping[str, object], raw_name_groups: frozenset[str] +) -> bool: + return ( + model_group is None + or model_group in raw_name_groups + or is_azure_model_router_request(model_group, hidden_params) + ) + + def _client_facing_response_model( - requested_model: str, hidden_params: Mapping[str, object], log_context: str + requested_model: str, + hidden_params: Mapping[str, object], + log_context: str, + raw_name_groups: frozenset[str], ) -> str | None: additional_headers: Final = as_str_mapping(hidden_params.get("additional_headers")) or _NO_HEADERS model_group: Final = _non_empty_str(additional_headers.get("x-litellm-model-group")) attempted_fallbacks: Final = additional_headers.get("x-litellm-attempted-fallbacks") if isinstance(attempted_fallbacks, (int, float)) and attempted_fallbacks > 0: + if _fallback_group_reports_served_model(model_group, hidden_params, raw_name_groups): + verbose_proxy_logger.debug( + "%s: fallback to model group=%r, preserving the model it served.", log_context, model_group + ) + return None verbose_proxy_logger.debug( "%s: fallback detected (attempted_fallbacks=%s), reporting fallback model group=%r.", log_context, @@ -1558,6 +1586,7 @@ def _override_openai_response_model( requested_model: str, log_context: str, return_raw_model_name: bool = False, + raw_name_groups: frozenset[str] = frozenset(), ) -> _ResponseT: """ Force the OpenAI-compatible `model` field in the response to match what the client requested. @@ -1576,8 +1605,10 @@ def _override_openai_response_model( Exceptions: 1. If a fallback occurred (indicated by x-litellm-attempted-fallbacks header), we report the fallback model group from the x-litellm-model-group header, the - same name streamed responses report, and leave the response unchanged when that - header is missing. Exceptions 2 and 3 do not apply to a fallback. + same name streamed responses report. The response keeps the model it served when + that header is missing, or when the group is an Azure Model Router or a complexity + router in ``raw_name_groups``, as a direct request to that group would. Exception 3 + does not apply to a fallback. 2. If the request was to an Azure Model Router, we preserve the actual model that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model. 3. If this was a fastest_response batch completion, use the winning model's @@ -1587,7 +1618,7 @@ def _override_openai_response_model( return response_obj target_model: Final = _client_facing_response_model( - requested_model, get_hidden_params_dict(response_obj), log_context + requested_model, get_hidden_params_dict(response_obj), log_context, raw_name_groups ) if target_model is None: return response_obj @@ -3332,6 +3363,7 @@ class ProxyBaseLLMRequestProcessing: requested_model=requested_model_from_client, log_context=f"litellm_call_id={logging_obj.litellm_call_id}", return_raw_model_name=self.returns_raw_model_name, + raw_name_groups=raw_name_router_groups(llm_router), ) fastapi_response.headers.update( diff --git a/tests/integration/authorization/test_responses_id_ownership.py b/tests/integration/authorization/test_responses_id_ownership.py index 2ab26e75f50..7750ff32776 100644 --- a/tests/integration/authorization/test_responses_id_ownership.py +++ b/tests/integration/authorization/test_responses_id_ownership.py @@ -196,7 +196,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway, "object": "response", "created_at": 1.0, "status": "completed", - "model": _MODEL, + "model": model, "output": [], "parallel_tool_calls": False, "tool_choice": "auto", @@ -216,7 +216,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway, "object": "response", "created_at": 1.0, "status": "cancelled", - "model": _MODEL, + "model": model, "output": [], "parallel_tool_calls": False, "tool_choice": "auto", @@ -385,7 +385,7 @@ def test_allow_unmanaged_response_ids_forward_the_vendor_id_on_create( "object": "response", "created_at": 1.0, "status": "completed", - "model": _MODEL, + "model": model, "output": [], "parallel_tool_calls": False, "tool_choice": "auto", diff --git a/tests/integration/providers/test_responses_lifecycle_wire.py b/tests/integration/providers/test_responses_lifecycle_wire.py index 6509c8ced96..596072b9920 100644 --- a/tests/integration/providers/test_responses_lifecycle_wire.py +++ b/tests/integration/providers/test_responses_lifecycle_wire.py @@ -175,7 +175,7 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response( "object": "response", "created_at": 1.0, "status": "completed", - "model": _MODEL, + "model": model, "output": [], }, retrieved.model_dump_json() requests: Final = drain_contract_requests(wire_b) @@ -279,7 +279,7 @@ def test_streamed_response_ids_are_stable_and_retrievable_on_each_alias( "object": "response", "created_at": 1.0, "status": "completed", - "model": _MODEL, + "model": model, "output": [], }, retrieved.model_dump_json() requests: Final = drain_contract_requests(wire) @@ -532,7 +532,7 @@ def test_cancel_forwards_provider_id_and_returns_cancelled_response( "object": "response", "created_at": 1.0, "status": "cancelled", - "model": _MODEL, + "model": model, "output": [], }, cancelled.model_dump_json() requests: Final = drain_contract_requests(wire) diff --git a/tests/integration/routing/test_fallback_retry_header_contracts.py b/tests/integration/routing/test_fallback_retry_header_contracts.py index ebed95a683c..08e8fd74d6a 100644 --- a/tests/integration/routing/test_fallback_retry_header_contracts.py +++ b/tests/integration/routing/test_fallback_retry_header_contracts.py @@ -170,7 +170,7 @@ def test_client_fallbacks_reach_an_allowed_model_and_name_a_denied_one(gateway: body: Final[dict[str, JsonValue]] = {"model": primary, "fallbacks": [fallback.name]} served: Final = _chat(gateway, body, key=scenario.key(models=[primary, fallback.name])) assert served.status_code == 200, served.text - assert _body(served)["model"] == fallback.upstream + assert _body(served)["model"] == fallback.name assert served.headers["x-litellm-model-id"] == fallback.deployment_id assert _content(served) denied: Final = _chat(gateway, body, key=scenario.key(models=[primary])) @@ -213,7 +213,7 @@ def test_request_fallbacks_reroute_after_a_connection_failure(gateway: Gateway) fallback: Final = _unique_model(scenario) response: Final = _chat(gateway, {"model": primary, "fallbacks": [fallback.name]}) assert response.status_code == 200, response.text - assert _body(response)["model"] == fallback.upstream + assert _body(response)["model"] == fallback.name assert response.headers["x-litellm-model-id"] == fallback.deployment_id assert response.headers["x-litellm-attempted-fallbacks"] == "1" diff --git a/tests/integration/routing/test_observed_routing.py b/tests/integration/routing/test_observed_routing.py index ca5f248e0de..788c8462e10 100644 --- a/tests/integration/routing/test_observed_routing.py +++ b/tests/integration/routing/test_observed_routing.py @@ -636,7 +636,7 @@ def test_health_routing_fallbacks_preserve_error_specific_targets_and_request_co "id": "response-" + expected_model, "object": "chat.completion", "created": 1, - "model": expected_model, + "model": "primary" if expected_model == "primary-tier2-up" else expected_model.removesuffix("-up"), "choices": [ { "index": 0, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 62ab7c64b1b..41a46616bf7 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -56,6 +56,7 @@ from litellm.proxy.common_request_processing import ( ttft_keepalive_interval, _override_openai_response_model, _parse_event_data_for_error, + raw_name_router_groups, _resolve_per_request_model_group_alias, should_return_raw_model_name, _sse_error_frames, @@ -3298,6 +3299,62 @@ class TestOverrideOpenAIResponseModel: assert stamped.model == "gpt-mini" + @pytest.mark.asyncio + @pytest.mark.parametrize("return_raw_model_name", [False, True]) + async def test_non_streamed_fallback_into_raw_name_router_keeps_the_served_model(self, return_raw_model_name): + router: Final = litellm.Router( + model_list=[ + {"model_name": "broken-primary", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}}, + {"model_name": "gpt-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}}, + { + "model_name": "auto-raw", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": {"SIMPLE": "gpt-mini", "MEDIUM": "gpt-mini", "COMPLEX": "gpt-mini"}, + "return_raw_model_name": return_raw_model_name, + }, + "complexity_router_default_model": "gpt-mini", + }, + }, + ], + fallbacks=[{"broken-primary": ["auto-raw"]}], + ) + response: Final = await router.acompletion( + model="broken-primary", + mock_testing_fallbacks=True, + mock_response="ok", + messages=[{"role": "user", "content": "hi"}], + ) + served_model: Final = response.model + + stamped: Final = _override_openai_response_model( + response_obj=response, + requested_model="broken-primary", + log_context="test_context", + raw_name_groups=raw_name_router_groups(router), + ) + + assert served_model not in ("auto-raw", "broken-primary") + assert stamped.model == (served_model if return_raw_model_name else "auto-raw") + + def test_non_streamed_fallback_into_azure_model_router_keeps_the_served_model(self): + from litellm.llms.azure_ai.common_utils import AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY + + served_model: Final = "azure_ai/gpt-5-nano-2025-08-07" + response_obj = MagicMock() + response_obj.model = served_model + response_obj._hidden_params = { + "additional_headers": {"x-litellm-attempted-fallbacks": 1, "x-litellm-model-group": "smart-pick"}, + AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: served_model, + } + + _override_openai_response_model( + response_obj=response_obj, requested_model="broken-primary", log_context="test_context" + ) + + assert response_obj.model == served_model + @pytest.mark.parametrize("requested_model", ["gpt-4", "azure_ai/model_router"]) def test_override_model_reports_fallback_group_when_fallback_occurred_object(self, requested_model): response_obj = MagicMock()