mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): keep the served model when a fallback lands on a raw-name or Azure Model Router group
This commit is contained in:
parent
7b3bd64445
commit
572f26b0cd
6 changed files with 102 additions and 13 deletions
|
|
@ -1522,13 +1522,41 @@ def _non_empty_str(value: object) -> str | None:
|
|||
_NO_HEADERS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def raw_name_router_groups(llm_router: Router | None) -> frozenset[str]:
|
||||
if llm_router is None:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
model_group
|
||||
for model_group, tagged_routers in llm_router.complexity_routers.items()
|
||||
if any(tagged.strategy.config.return_raw_model_name for tagged in tagged_routers)
|
||||
)
|
||||
|
||||
|
||||
def _fallback_group_reports_served_model(
|
||||
model_group: str | None, hidden_params: Mapping[str, object], raw_name_groups: frozenset[str]
|
||||
) -> bool:
|
||||
return (
|
||||
model_group is None
|
||||
or model_group in raw_name_groups
|
||||
or is_azure_model_router_request(model_group, hidden_params)
|
||||
)
|
||||
|
||||
|
||||
def _client_facing_response_model(
|
||||
requested_model: str, hidden_params: Mapping[str, object], log_context: str
|
||||
requested_model: str,
|
||||
hidden_params: Mapping[str, object],
|
||||
log_context: str,
|
||||
raw_name_groups: frozenset[str],
|
||||
) -> str | None:
|
||||
additional_headers: Final = as_str_mapping(hidden_params.get("additional_headers")) or _NO_HEADERS
|
||||
model_group: Final = _non_empty_str(additional_headers.get("x-litellm-model-group"))
|
||||
attempted_fallbacks: Final = additional_headers.get("x-litellm-attempted-fallbacks")
|
||||
if isinstance(attempted_fallbacks, (int, float)) and attempted_fallbacks > 0:
|
||||
if _fallback_group_reports_served_model(model_group, hidden_params, raw_name_groups):
|
||||
verbose_proxy_logger.debug(
|
||||
"%s: fallback to model group=%r, preserving the model it served.", log_context, model_group
|
||||
)
|
||||
return None
|
||||
verbose_proxy_logger.debug(
|
||||
"%s: fallback detected (attempted_fallbacks=%s), reporting fallback model group=%r.",
|
||||
log_context,
|
||||
|
|
@ -1558,6 +1586,7 @@ def _override_openai_response_model(
|
|||
requested_model: str,
|
||||
log_context: str,
|
||||
return_raw_model_name: bool = False,
|
||||
raw_name_groups: frozenset[str] = frozenset(),
|
||||
) -> _ResponseT:
|
||||
"""
|
||||
Force the OpenAI-compatible `model` field in the response to match what the client requested.
|
||||
|
|
@ -1576,8 +1605,10 @@ def _override_openai_response_model(
|
|||
Exceptions:
|
||||
1. If a fallback occurred (indicated by x-litellm-attempted-fallbacks header),
|
||||
we report the fallback model group from the x-litellm-model-group header, the
|
||||
same name streamed responses report, and leave the response unchanged when that
|
||||
header is missing. Exceptions 2 and 3 do not apply to a fallback.
|
||||
same name streamed responses report. The response keeps the model it served when
|
||||
that header is missing, or when the group is an Azure Model Router or a complexity
|
||||
router in ``raw_name_groups``, as a direct request to that group would. Exception 3
|
||||
does not apply to a fallback.
|
||||
2. If the request was to an Azure Model Router, we preserve the actual model
|
||||
that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model.
|
||||
3. If this was a fastest_response batch completion, use the winning model's
|
||||
|
|
@ -1587,7 +1618,7 @@ def _override_openai_response_model(
|
|||
return response_obj
|
||||
|
||||
target_model: Final = _client_facing_response_model(
|
||||
requested_model, get_hidden_params_dict(response_obj), log_context
|
||||
requested_model, get_hidden_params_dict(response_obj), log_context, raw_name_groups
|
||||
)
|
||||
if target_model is None:
|
||||
return response_obj
|
||||
|
|
@ -3332,6 +3363,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
requested_model=requested_model_from_client,
|
||||
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
|
||||
return_raw_model_name=self.returns_raw_model_name,
|
||||
raw_name_groups=raw_name_router_groups(llm_router),
|
||||
)
|
||||
|
||||
fastapi_response.headers.update(
|
||||
|
|
|
|||
|
|
@ -196,7 +196,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway,
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "completed",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
|
|
@ -216,7 +216,7 @@ def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway,
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "cancelled",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
|
|
@ -385,7 +385,7 @@ def test_allow_unmanaged_response_ids_forward_the_vendor_id_on_create(
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "completed",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response(
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "completed",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
}, retrieved.model_dump_json()
|
||||
requests: Final = drain_contract_requests(wire_b)
|
||||
|
|
@ -279,7 +279,7 @@ def test_streamed_response_ids_are_stable_and_retrievable_on_each_alias(
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "completed",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
}, retrieved.model_dump_json()
|
||||
requests: Final = drain_contract_requests(wire)
|
||||
|
|
@ -532,7 +532,7 @@ def test_cancel_forwards_provider_id_and_returns_cancelled_response(
|
|||
"object": "response",
|
||||
"created_at": 1.0,
|
||||
"status": "cancelled",
|
||||
"model": _MODEL,
|
||||
"model": model,
|
||||
"output": [],
|
||||
}, cancelled.model_dump_json()
|
||||
requests: Final = drain_contract_requests(wire)
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ def test_client_fallbacks_reach_an_allowed_model_and_name_a_denied_one(gateway:
|
|||
body: Final[dict[str, JsonValue]] = {"model": primary, "fallbacks": [fallback.name]}
|
||||
served: Final = _chat(gateway, body, key=scenario.key(models=[primary, fallback.name]))
|
||||
assert served.status_code == 200, served.text
|
||||
assert _body(served)["model"] == fallback.upstream
|
||||
assert _body(served)["model"] == fallback.name
|
||||
assert served.headers["x-litellm-model-id"] == fallback.deployment_id
|
||||
assert _content(served)
|
||||
denied: Final = _chat(gateway, body, key=scenario.key(models=[primary]))
|
||||
|
|
@ -213,7 +213,7 @@ def test_request_fallbacks_reroute_after_a_connection_failure(gateway: Gateway)
|
|||
fallback: Final = _unique_model(scenario)
|
||||
response: Final = _chat(gateway, {"model": primary, "fallbacks": [fallback.name]})
|
||||
assert response.status_code == 200, response.text
|
||||
assert _body(response)["model"] == fallback.upstream
|
||||
assert _body(response)["model"] == fallback.name
|
||||
assert response.headers["x-litellm-model-id"] == fallback.deployment_id
|
||||
assert response.headers["x-litellm-attempted-fallbacks"] == "1"
|
||||
|
||||
|
|
|
|||
|
|
@ -636,7 +636,7 @@ def test_health_routing_fallbacks_preserve_error_specific_targets_and_request_co
|
|||
"id": "response-" + expected_model,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": expected_model,
|
||||
"model": "primary" if expected_model == "primary-tier2-up" else expected_model.removesuffix("-up"),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ from litellm.proxy.common_request_processing import (
|
|||
ttft_keepalive_interval,
|
||||
_override_openai_response_model,
|
||||
_parse_event_data_for_error,
|
||||
raw_name_router_groups,
|
||||
_resolve_per_request_model_group_alias,
|
||||
should_return_raw_model_name,
|
||||
_sse_error_frames,
|
||||
|
|
@ -3298,6 +3299,62 @@ class TestOverrideOpenAIResponseModel:
|
|||
|
||||
assert stamped.model == "gpt-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("return_raw_model_name", [False, True])
|
||||
async def test_non_streamed_fallback_into_raw_name_router_keeps_the_served_model(self, return_raw_model_name):
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "broken-primary", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}},
|
||||
{"model_name": "gpt-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-fake"}},
|
||||
{
|
||||
"model_name": "auto-raw",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"tiers": {"SIMPLE": "gpt-mini", "MEDIUM": "gpt-mini", "COMPLEX": "gpt-mini"},
|
||||
"return_raw_model_name": return_raw_model_name,
|
||||
},
|
||||
"complexity_router_default_model": "gpt-mini",
|
||||
},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"broken-primary": ["auto-raw"]}],
|
||||
)
|
||||
response: Final = await router.acompletion(
|
||||
model="broken-primary",
|
||||
mock_testing_fallbacks=True,
|
||||
mock_response="ok",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
served_model: Final = response.model
|
||||
|
||||
stamped: Final = _override_openai_response_model(
|
||||
response_obj=response,
|
||||
requested_model="broken-primary",
|
||||
log_context="test_context",
|
||||
raw_name_groups=raw_name_router_groups(router),
|
||||
)
|
||||
|
||||
assert served_model not in ("auto-raw", "broken-primary")
|
||||
assert stamped.model == (served_model if return_raw_model_name else "auto-raw")
|
||||
|
||||
def test_non_streamed_fallback_into_azure_model_router_keeps_the_served_model(self):
|
||||
from litellm.llms.azure_ai.common_utils import AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY
|
||||
|
||||
served_model: Final = "azure_ai/gpt-5-nano-2025-08-07"
|
||||
response_obj = MagicMock()
|
||||
response_obj.model = served_model
|
||||
response_obj._hidden_params = {
|
||||
"additional_headers": {"x-litellm-attempted-fallbacks": 1, "x-litellm-model-group": "smart-pick"},
|
||||
AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: served_model,
|
||||
}
|
||||
|
||||
_override_openai_response_model(
|
||||
response_obj=response_obj, requested_model="broken-primary", log_context="test_context"
|
||||
)
|
||||
|
||||
assert response_obj.model == served_model
|
||||
|
||||
@pytest.mark.parametrize("requested_model", ["gpt-4", "azure_ai/model_router"])
|
||||
def test_override_model_reports_fallback_group_when_fallback_occurred_object(self, requested_model):
|
||||
response_obj = MagicMock()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue