From 27ec8d2adf840aaa2e9c7451a372053b03b64273 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:26 -0700 Subject: [PATCH] fix(router): count a failed deployment as tried when the caller supplies the key With caller-supplied credentials the retry record's deployment_id is the generated id, so the web search preference never saw the configured deployment that failed and sent every retry back to it. Retry records now also carry the configured id. --- litellm/router.py | 13 ++++++++++--- litellm/router_utils/common_utils.py | 4 +++- litellm/types/router.py | 3 ++- .../test_router_utils_common_utils.py | 15 ++++++++++++--- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 16eadfe159f..ec0692a937f 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -119,6 +119,7 @@ from litellm.litellm_core_utils.batch_utils import ( ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, coerce_token_limit, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, @@ -9880,14 +9881,15 @@ class Router: _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") - model_info: Final = request_metadata.get("model_info") - deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {} + deployment_id: Final = model_info.get("id") + configured_deployment_id: Final = model_info.get("original_model_id") base_model: Final = get_base_model_from_litellm_call_metadata( metadata=request_metadata ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") - attempt_record: Final[RetryAttemptRecord] = { + attempt_fields: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, "model": deployment_model if isinstance(deployment_model, str) else None, @@ -9896,6 +9898,11 @@ class Router: "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, } + attempt_record: Final[RetryAttemptRecord] = ( + {**attempt_fields, "configured_deployment_id": configured_deployment_id} + if isinstance(configured_deployment_id, str) + else attempt_fields + ) earlier_breadcrumbs: Final = request_metadata.get("previous_models") kept_breadcrumbs: Final[tuple[object, ...]] = ( tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 5b3f923d936..34e12c1f751 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -515,7 +515,9 @@ def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) -> *_retry_breadcrumbs(request_kwargs.get("litellm_metadata")), ) return tuple( - deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str) + deployment_id + for breadcrumb in breadcrumbs + if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str) ) diff --git a/litellm/types/router.py b/litellm/types/router.py index f8de2128a80..75c4eea9547 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -23,7 +23,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator -from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable +from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -1197,6 +1197,7 @@ class RetryAttemptRecord(TypedDict): exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] + configured_deployment_id: ReadOnly[NotRequired[str]] VALID_LITELLM_ENVIRONMENTS = [ diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0049793a354..a28731afad1 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -735,8 +735,12 @@ async def test_web_search_preference_gives_way_once_the_capable_deployment_faile @pytest.mark.asyncio @_WEB_SEARCH_REQUESTS +@pytest.mark.parametrize( + "caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"] +) async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one( web_search_request: dict[str, object], + caller_credentials: dict[str, str], ) -> None: router: Final = Router( model_list=[ @@ -752,7 +756,12 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches }, { "model_name": "mixed", - "litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6}, + "litellm_params": { + "model": "openai/gpt-y", + "api_key": "fake-key", + "mock_response": "from no-search", + "weight": 10**6, + }, "model_info": {"id": "no-search", "supports_web_search": False}, }, ], @@ -760,9 +769,9 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches disable_cooldowns=True, ) response: Final = await router.acompletion( - model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request + model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials ) - assert response._hidden_params["model_id"] == "no-search" + assert response.choices[0].message.content == "from no-search" def _rate_limited_web_search_group(routing_strategy: str) -> Router: