fix(router): count a failed deployment as tried when the caller supplies the key

With caller-supplied credentials the retry record's deployment_id is the generated
id, so the web search preference never saw the configured deployment that failed
and sent every retry back to it. Retry records now also carry the configured id.
This commit is contained in:
mateo-berri 2026-10-29 17:04:26 -07:00
parent a179118b3e
commit 27ec8d2adf
4 changed files with 27 additions and 8 deletions

View file

@ -119,6 +119,7 @@ from litellm.litellm_core_utils.batch_utils import (
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
coerce_token_limit,
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -9880,14 +9881,15 @@ class Router:
_metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs
model_group: Final = kwargs.get("model")
model_info: Final = request_metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None
model_info: Final = as_str_mapping(request_metadata.get("model_info")) or {}
deployment_id: Final = model_info.get("id")
configured_deployment_id: Final = model_info.get("original_model_id")
base_model: Final = get_base_model_from_litellm_call_metadata(
metadata=request_metadata
) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None)
deployment_model: Final = request_metadata.get("deployment")
attempted_retries: Final = request_metadata.get("attempted_retries")
attempt_record: Final[RetryAttemptRecord] = {
attempt_fields: Final[RetryAttemptRecord] = {
"model_group": model_group if isinstance(model_group, str) else None,
"deployment_id": deployment_id if isinstance(deployment_id, str) else None,
"model": deployment_model if isinstance(deployment_model, str) else None,
@ -9896,6 +9898,11 @@ class Router:
"exception_string": str(e),
"attempted_retries": attempted_retries if type(attempted_retries) is int else None,
}
attempt_record: Final[RetryAttemptRecord] = (
{**attempt_fields, "configured_deployment_id": configured_deployment_id}
if isinstance(configured_deployment_id, str)
else attempt_fields
)
earlier_breadcrumbs: Final = request_metadata.get("previous_models")
kept_breadcrumbs: Final[tuple[object, ...]] = (
tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :]

View file

@ -515,7 +515,9 @@ def _deployment_ids_retried_in_request(request_kwargs: Mapping[str, object]) ->
*_retry_breadcrumbs(request_kwargs.get("litellm_metadata")),
)
return tuple(
deployment_id for breadcrumb in breadcrumbs if isinstance(deployment_id := breadcrumb.get("deployment_id"), str)
deployment_id
for breadcrumb in breadcrumbs
if isinstance(deployment_id := breadcrumb.get("configured_deployment_id", breadcrumb.get("deployment_id")), str)
)

View file

@ -23,7 +23,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import httpx
from openai import Timeout as SDKTimeout
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -1197,6 +1197,7 @@ class RetryAttemptRecord(TypedDict):
exception_type: ReadOnly[str]
exception_string: ReadOnly[str]
attempted_retries: ReadOnly[int | None]
configured_deployment_id: ReadOnly[NotRequired[str]]
VALID_LITELLM_ENVIRONMENTS = [

View file

@ -735,8 +735,12 @@ async def test_web_search_preference_gives_way_once_the_capable_deployment_faile
@pytest.mark.asyncio
@_WEB_SEARCH_REQUESTS
@pytest.mark.parametrize(
"caller_credentials", [{}, {"api_key": "caller-key"}], ids=["configured-credentials", "caller-credentials"]
)
async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches_the_other_one(
web_search_request: dict[str, object],
caller_credentials: dict[str, str],
) -> None:
router: Final = Router(
model_list=[
@ -752,7 +756,12 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches
},
{
"model_name": "mixed",
"litellm_params": {"model": "openai/gpt-y", "api_key": "fake-key", "mock_response": "hi", "weight": 10**6},
"litellm_params": {
"model": "openai/gpt-y",
"api_key": "fake-key",
"mock_response": "from no-search",
"weight": 10**6,
},
"model_info": {"id": "no-search", "supports_web_search": False},
},
],
@ -760,9 +769,9 @@ async def test_retry_after_a_retryable_failure_on_the_capable_deployment_reaches
disable_cooldowns=True,
)
response: Final = await router.acompletion(
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request
model="mixed", messages=[{"role": "user", "content": "hi"}], **web_search_request, **caller_credentials
)
assert response._hidden_params["model_id"] == "no-search"
assert response.choices[0].message.content == "from no-search"
def _rate_limited_web_search_group(routing_strategy: str) -> Router: