fix(proxy): keep the served model when a fallback lands on a raw-name or Azure Model Router group

This commit is contained in:
mateo-berri 2026-10-29 09:28:51 -07:00
parent 7b3bd64445
commit 572f26b0cd
6 changed files with 102 additions and 13 deletions

View file

@ -1522,13 +1522,41 @@ def _non_empty_str(value: object) -> str | None:
_NO_HEADERS: Final[Mapping[str, object]] = MappingProxyType({})
def raw_name_router_groups(llm_router: Router | None) -> frozenset[str]:
if llm_router is None:
return frozenset()
return frozenset(
model_group
for model_group, tagged_routers in llm_router.complexity_routers.items()
if any(tagged.strategy.config.return_raw_model_name for tagged in tagged_routers)
)
def _fallback_group_reports_served_model(
model_group: str | None, hidden_params: Mapping[str, object], raw_name_groups: frozenset[str]
) -> bool:
return (
model_group is None
or model_group in raw_name_groups
or is_azure_model_router_request(model_group, hidden_params)
)
def _client_facing_response_model(
requested_model: str, hidden_params: Mapping[str, object], log_context: str
requested_model: str,
hidden_params: Mapping[str, object],
log_context: str,
raw_name_groups: frozenset[str],
) -> str | None:
additional_headers: Final = as_str_mapping(hidden_params.get("additional_headers")) or _NO_HEADERS
model_group: Final = _non_empty_str(additional_headers.get("x-litellm-model-group"))
attempted_fallbacks: Final = additional_headers.get("x-litellm-attempted-fallbacks")
if isinstance(attempted_fallbacks, (int, float)) and attempted_fallbacks > 0:
if _fallback_group_reports_served_model(model_group, hidden_params, raw_name_groups):
verbose_proxy_logger.debug(
"%s: fallback to model group=%r, preserving the model it served.", log_context, model_group
)
return None
verbose_proxy_logger.debug(
"%s: fallback detected (attempted_fallbacks=%s), reporting fallback model group=%r.",
log_context,
@ -1558,6 +1586,7 @@ def _override_openai_response_model(
requested_model: str,
log_context: str,
return_raw_model_name: bool = False,
raw_name_groups: frozenset[str] = frozenset(),
) -> _ResponseT:
"""
Force the OpenAI-compatible `model` field in the response to match what the client requested.
@ -1576,8 +1605,10 @@ def _override_openai_response_model(
Exceptions:
1. If a fallback occurred (indicated by x-litellm-attempted-fallbacks header),
we report the fallback model group from the x-litellm-model-group header, the
same name streamed responses report, and leave the response unchanged when that
header is missing. Exceptions 2 and 3 do not apply to a fallback.
same name streamed responses report. The response keeps the model it served when
that header is missing, or when the group is an Azure Model Router or a complexity
router in ``raw_name_groups``, as a direct request to that group would. Exception 3
does not apply to a fallback.
2. If the request was to an Azure Model Router, we preserve the actual model
that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model.
3. If this was a fastest_response batch completion, use the winning model's
@ -1587,7 +1618,7 @@ def _override_openai_response_model(
return response_obj
target_model: Final = _client_facing_response_model(
requested_model, get_hidden_params_dict(response_obj), log_context
requested_model, get_hidden_params_dict(response_obj), log_context, raw_name_groups
)
if target_model is None:
return response_obj
@ -3332,6 +3363,7 @@ class ProxyBaseLLMRequestProcessing:
requested_model=requested_model_from_client,
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=self.returns_raw_model_name,
raw_name_groups=raw_name_router_groups(llm_router),
)
fastapi_response.headers.update(

View file

@ -196,7 +196,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway,
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": _MODEL,
"model": model,
"output": [],
"parallel_tool_calls": False,
"tool_choice": "auto",
@ -216,7 +216,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway,
"object": "response",
"created_at": 1.0,
"status": "cancelled",
"model": _MODEL,
"model": model,
"output": [],
"parallel_tool_calls": False,
"tool_choice": "auto",
@ -385,7 +385,7 @@ def test_allow_unmanaged_response_ids_forward_the_vendor_id_on_create(
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": _MODEL,
"model": model,
"output": [],
"parallel_tool_calls": False,
"tool_choice": "auto",

View file

@ -175,7 +175,7 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response(
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": _MODEL,
"model": model,
"output": [],
}, retrieved.model_dump_json()
requests: Final = drain_contract_requests(wire_b)
@ -279,7 +279,7 @@ def test_streamed_response_ids_are_stable_and_retrievable_on_each_alias(
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": _MODEL,
"model": model,
"output": [],
}, retrieved.model_dump_json()
requests: Final = drain_contract_requests(wire)
@ -532,7 +532,7 @@ def test_cancel_forwards_provider_id_and_returns_cancelled_response(
"object": "response",
"created_at": 1.0,
"status": "cancelled",
"model": _MODEL,
"model": model,
"output": [],
}, cancelled.model_dump_json()
requests: Final = drain_contract_requests(wire)

View file

@ -170,7 +170,7 @@ def test_client_fallbacks_reach_an_allowed_model_and_name_a_denied_one(gateway:
body: Final[dict[str, JsonValue]] = {"model": primary, "fallbacks": [fallback.name]}
served: Final = _chat(gateway, body, key=scenario.key(models=[primary, fallback.name]))
assert served.status_code == 200, served.text
assert _body(served)["model"] == fallback.upstream
assert _body(served)["model"] == fallback.name
assert served.headers["x-litellm-model-id"] == fallback.deployment_id
assert _content(served)
denied: Final = _chat(gateway, body, key=scenario.key(models=[primary]))
@ -213,7 +213,7 @@ def test_request_fallbacks_reroute_after_a_connection_failure(gateway: Gateway)
fallback: Final = _unique_model(scenario)
response: Final = _chat(gateway, {"model": primary, "fallbacks": [fallback.name]})
assert response.status_code == 200, response.text
assert _body(response)["model"] == fallback.upstream
assert _body(response)["model"] == fallback.name
assert response.headers["x-litellm-model-id"] == fallback.deployment_id
assert response.headers["x-litellm-attempted-fallbacks"] == "1"

View file

@ -636,7 +636,7 @@ def test_health_routing_fallbacks_preserve_error_specific_targets_and_request_co
"id": "response-" + expected_model,
"object": "chat.completion",
"created": 1,
"model": expected_model,
"model": "primary" if expected_model == "primary-tier2-up" else expected_model.removesuffix("-up"),
"choices": [
{
"index": 0,

View file

@ -56,6 +56,7 @@ from litellm.proxy.common_request_processing import (
ttft_keepalive_interval,
_override_openai_response_model,
_parse_event_data_for_error,
raw_name_router_groups,
_resolve_per_request_model_group_alias,
should_return_raw_model_name,
_sse_error_frames,
@ -3298,6 +3299,62 @@ class TestOverrideOpenAIResponseModel:
assert stamped.model == "gpt-mini"
@pytest.mark.asyncio
@pytest.mark.parametrize("return_raw_model_name", [False, True])
async def test_non_streamed_fallback_into_raw_name_router_keeps_the_served_model(self, return_raw_model_name):
router: Final = litellm.Router(
model_list=[
{"model_name": "broken-primary", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}},
{"model_name": "gpt-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}},
{
"model_name": "auto-raw",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {"SIMPLE": "gpt-mini", "MEDIUM": "gpt-mini", "COMPLEX": "gpt-mini"},
"return_raw_model_name": return_raw_model_name,
},
"complexity_router_default_model": "gpt-mini",
},
},
],
fallbacks=[{"broken-primary": ["auto-raw"]}],
)
response: Final = await router.acompletion(
model="broken-primary",
mock_testing_fallbacks=True,
mock_response="ok",
messages=[{"role": "user", "content": "hi"}],
)
served_model: Final = response.model
stamped: Final = _override_openai_response_model(
response_obj=response,
requested_model="broken-primary",
log_context="test_context",
raw_name_groups=raw_name_router_groups(router),
)
assert served_model not in ("auto-raw", "broken-primary")
assert stamped.model == (served_model if return_raw_model_name else "auto-raw")
def test_non_streamed_fallback_into_azure_model_router_keeps_the_served_model(self):
from litellm.llms.azure_ai.common_utils import AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY
served_model: Final = "azure_ai/gpt-5-nano-2025-08-07"
response_obj = MagicMock()
response_obj.model = served_model
response_obj._hidden_params = {
"additional_headers": {"x-litellm-attempted-fallbacks": 1, "x-litellm-model-group": "smart-pick"},
AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: served_model,
}
_override_openai_response_model(
response_obj=response_obj, requested_model="broken-primary", log_context="test_context"
)
assert response_obj.model == served_model
@pytest.mark.parametrize("requested_model", ["gpt-4", "azure_ai/model_router"])
def test_override_model_reports_fallback_group_when_fallback_occurred_object(self, requested_model):
response_obj = MagicMock()