diff --git a/litellm/router.py b/litellm/router.py index 16eadfe159f..ec0692a937f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -119,6 +119,7 @@ from litellm.litellm_core_utils.batch_utils import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -9880,14 +9881,15 @@ class Router: _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") - model_info: Final = request_metadata.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + configured_deployment_id: Final = model_info.get("original_model_id") base_model: Final = get_base_model_from_litellm_call_metadata( metadata=request_metadata ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") - attempt_record: Final[RetryAttemptRecord] = { + attempt_fields: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, "model": deployment_model if isinstance(deployment_model, str) else None, @@ -9896,6 +9898,11 @@ class Router: "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, } + attempt_record: Final[RetryAttemptRecord] = ( + {**attempt_fields, "configured_deployment_id": configured_deployment_id} + if isinstance(configured_deployment_id, str) + else attempt_fields + ) earlier_breadcrumbs: Final = request_metadata.get("previous_models") kept_breadcrumbs: Final[tuple[object, ...]] = ( tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 5b3f923d936..34e12c1f751 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -515,7 +515,9 @@ def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), ) return tuple( - deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str) + deployment_id + for breadcrumb in breadcrumbs + if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str) ) diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..75c4eea9547 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -23,7 +23,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator -from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable +from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -1197,6 +1197,7 @@ class RetryAttemptRecord(TypedDict): exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] + configured_deployment_id: ReadOnly[NotRequired[str]] VALID_LITELLM_ENVIRONMENTS = [ diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0049793a354..a28731afad1 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -735,8 +735,12 @@ async def test_web_search_preference_gives_way_once_the_capable_deployment_faile @pytest.mark.asyncio @_WEB_SEARCH_REQUESTS +@pytest.mark.parametrize( + "caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"] +) async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( web_search_request: dict[str, object], + caller_credentials: dict[str, str], ) -> None: router: Final = Router( model_list=[ @@ -752,7 +756,12 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches }, { "model_name": "mixed", - "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6}, + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, "model_info": {"id": "no-search", "supports_web_search": False}, }, ], @@ -760,9 +769,9 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches disable_cooldowns=True, ) response: Final = await router.acompletion( - model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials ) - assert response._hidden_params["model_id"] == "no-search" + assert response.choices[0].message.content == "from no-search" def _rate_limited_web_search_group(routing_strategy: str) -> Router: