diff --git a/litellm/router.py b/litellm/router.py index fc9743ff43d..d57106749f6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3202,6 +3202,17 @@ class Router: ): await response.fetch_stream() + # Record that this deployment successfully served this model. + # Stored in Redis (via DualCache) so all router instances share the signal. + _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id") if deployment else None + if _dep_id_cap: + asyncio.create_task( + self.cache.async_set_cache( + key=f"litellm:cap:{_dep_id_cap}:{model}", + value=True, + ttl=86400, # 24h — re-confirm capability daily + ) + ) self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # debug how often this deployment picked @@ -3239,6 +3250,19 @@ class Router: if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs) + # 404 → provider does not support this model. Cache this signal in Redis + # (via DualCache) so every router instance skips this deployment next time. + # TTL of 1h means we re-test after an hour, in case the provider adds support. + if isinstance(e, litellm.NotFoundError): + _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id") + if _dep_id_cap: + asyncio.create_task( + self.cache.async_set_cache( + key=f"litellm:cap:{_dep_id_cap}:{model}", + value=False, + ttl=3600, # 1h — re-test after TTL + ) + ) raise e def _update_kwargs_before_fallbacks( @@ -7330,11 +7354,15 @@ class Router: status_code: Final = getattr(error, "status_code", None) if status_code is not None and not litellm._should_retry(status_code): # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403): + # 404 means the provider does not serve this model, so the retry loop is + # allowed to advance to the next-priority deployment. The + # `_num_healthy_deployments <= 0` guard at the end of this method still + # raises once nothing else is left to try, and the capability cache in + # async_get_healthy_deployments keeps the 404-ing deployment out of the + # candidate list on subsequent attempts. + if status_code not in (401, 403, 404): raise error - if isinstance(error, litellm.NotFoundError): - raise error # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): if ( @@ -11714,21 +11742,46 @@ class Router: request_kwargs=request_kwargs, ) - ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order - ) + ## CAPABILITY CACHE FILTER ## -> skip deployments whose 404 responses have been + ## cached, indicating they cannot serve this model at this provider. The cache + ## is backed by Redis (via self.cache DualCache) with a TTL so providers that + ## later add model support are automatically re-tested after expiry. + ## key schema: litellm:cap:: + ## None → unknown (first request) → allow + ## True → confirmed capable → allow + ## False → confirmed incapable (404) → skip + _capable_deployments: list[dict] = [] + for _dep in cast(list[dict], healthy_deployments): + _dep_id: str | None = (_dep.get("model_info") or {}).get("id") + if _dep_id: + _cap_key = f"litellm:cap:{_dep_id}:{model}" + _cached_cap = await self.cache.async_get_cache(_cap_key) + if _cached_cap is not False: # None or True → keep + _capable_deployments.append(_dep) + else: + _capable_deployments.append(_dep) # no id to look up, allow through + if _capable_deployments: # only narrow if filter leaves at least one candidate + healthy_deployments = _capable_deployments - ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in - ## this request via weighted-failover. Always honored, regardless of the - ## router-level flag, so a stale exclusion key on kwargs cannot escape. + ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in this + ## request via weighted-failover. Runs BEFORE order filter so that excluded + ## order-1 deployments don't prevent order-2 from being selected — this was + ## the root cause of silent 404 routing failures with order-based fallback. _excluded_deployment_ids: Final = (request_kwargs or {}).pop("_excluded_deployment_ids", None) healthy_deployments = litellm.utils._get_excluded_filtered_deployments( cast(list[dict], healthy_deployments), excluded_deployment_ids=_excluded_deployment_ids, ) + ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments + ## with the lowest order value (order=1 preferred over order=2). Runs AFTER + ## exclusion so incapable/excluded order-1 deployments correctly fall through + ## to order-2 rather than producing an empty deployment list. + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + healthy_deployments = litellm.utils._get_order_filtered_deployments( + cast(list[dict], healthy_deployments), target_order=_target_order + ) + if len(healthy_deployments) == 0: exception: Final = await async_raise_no_deployment_exception( litellm_router_instance=self, diff --git a/tests/test_litellm/test_router_capability_cache.py b/tests/test_litellm/test_router_capability_cache.py new file mode 100644 index 00000000000..91a2717a095 --- /dev/null +++ b/tests/test_litellm/test_router_capability_cache.py @@ -0,0 +1,212 @@ +"""Tests for the router capability cache and the order/exclusion filter ordering. + +Covers three behaviours introduced together: + +1. A provider answering 404 for a model no longer aborts the request. The retry + loop is allowed to advance to the next-priority deployment. +2. `async_get_healthy_deployments` applies weighted-failover exclusion BEFORE the + `model_info.order` filter, so excluded order-1 deployments fall through to + order-2 instead of emptying the candidate list. +3. A DualCache entry per (deployment id, model group) records which providers + answered 404, so later requests skip them until the TTL expires. +""" + +import pytest + +import litellm +from litellm import Router + +MODEL_GROUP = "universal-group" + + +def _two_tier_router() -> Router: + """Two deployments in one group: order 1 and order 2.""" + return Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/tier-one", "api_key": "sk-tier-one"}, + "model_info": {"id": "dep-order-1", "order": 1}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/tier-two", "api_key": "sk-tier-two"}, + "model_info": {"id": "dep-order-2", "order": 2}, + }, + ] + ) + + +def _flat_router() -> Router: + """Two deployments in one group, no order set.""" + return Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/alpha", "api_key": "sk-alpha"}, + "model_info": {"id": "dep-alpha"}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/beta", "api_key": "sk-beta"}, + "model_info": {"id": "dep-beta"}, + }, + ] + ) + + +def _not_found() -> litellm.NotFoundError: + return litellm.NotFoundError( + message="The model 'some-model' does not exist", + model=MODEL_GROUP, + llm_provider="openai", + ) + + +def _ids(deployments) -> set: + return {d["model_info"]["id"] for d in deployments} + + +# ---------------------------------------------------------------- retry gating + + +def test_404_retries_while_other_deployments_remain(): + """A 404 returns True so the retry loop moves to the next deployment.""" + router = _two_tier_router() + + assert ( + router.should_retry_this_error( + error=_not_found(), + healthy_deployments=[{"model_info": {"id": "dep-order-2"}}], + all_deployments=router.model_list, + ) + is True + ) + + +def test_404_raises_once_nothing_healthy_is_left(): + """The last 404 still surfaces to the caller instead of retrying forever.""" + router = _two_tier_router() + + with pytest.raises(litellm.NotFoundError): + router.should_retry_this_error( + error=_not_found(), + healthy_deployments=[], + all_deployments=router.model_list, + ) + + +@pytest.mark.parametrize( + "error", + [ + litellm.BadRequestError(message="bad", model=MODEL_GROUP, llm_provider="openai"), + litellm.ContextWindowExceededError( + message="too long", model=MODEL_GROUP, llm_provider="openai" + ), + ], + ids=["400", "context-window"], +) +def test_other_4xx_errors_still_raise(error): + """Only 404 was exempted; other non-retryable 4xx keep aborting the request.""" + router = _two_tier_router() + + with pytest.raises(type(error)): + router.should_retry_this_error( + error=error, + healthy_deployments=[{"model_info": {"id": "dep-order-2"}}], + all_deployments=router.model_list, + ) + + +# ------------------------------------------------------- exclusion before order + + +@pytest.mark.asyncio +async def test_excluded_order_1_falls_through_to_order_2(): + """Regression: excluding every order-1 deployment used to empty the list. + + The order filter narrowed to order-1 first, exclusion then removed all of + them, and the caller saw "No deployments available" while a healthy order-2 + deployment was sitting right there. + """ + router = _two_tier_router() + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={"_excluded_deployment_ids": ["dep-order-1"]}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-order-2"} + + +@pytest.mark.asyncio +async def test_order_1_preferred_when_nothing_is_excluded(): + """The order filter still wins when exclusion removes nothing.""" + router = _two_tier_router() + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-order-1"} + + +# --------------------------------------------------------------- capability cache + + +@pytest.mark.asyncio +async def test_incapable_deployment_is_skipped(): + """A deployment cached as incapable drops out of the candidate list.""" + router = _flat_router() + await router.cache.async_set_cache( + key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=False, ttl=3600 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-beta"} + + +@pytest.mark.asyncio +async def test_capable_and_unknown_deployments_are_kept(): + """True means keep; a missing entry means untested, so also keep.""" + router = _flat_router() + await router.cache.async_set_cache( + key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=True, ttl=86400 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-alpha", "dep-beta"} + + +@pytest.mark.asyncio +async def test_all_incapable_falls_back_to_the_full_list(): + """With every candidate cached incapable the filter is skipped. + + Better to re-probe a stale cache than to refuse a request outright. + """ + router = _flat_router() + for dep_id in ("dep-alpha", "dep-beta"): + await router.cache.async_set_cache( + key=f"litellm:cap:{dep_id}:{MODEL_GROUP}", value=False, ttl=3600 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-alpha", "dep-beta"}