Merge 940f57b3b8 into 176b2e5eb8

litellm/router.py

@@ -3202,6 +3202,17 @@ class Router:
             ):
                 await response.fetch_stream()
 
+            # Record that this deployment successfully served this model.
+            # Stored in Redis (via DualCache) so all router instances share the signal.
+            _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id") if deployment else None
+            if _dep_id_cap:
+                asyncio.create_task(
+                    self.cache.async_set_cache(
+                        key=f"litellm:cap:{_dep_id_cap}:{model}",
+                        value=True,
+                        ttl=86400,  # 24h — re-confirm capability daily
+                    )
+                )
             self.success_calls[model_name] += 1
             verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
             # debug how often this deployment picked
@@ -3239,6 +3250,19 @@ class Router:
             if deployment is not None:
                 self._set_deployment_num_retries_on_exception(e, deployment)
                 self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs)
+                # 404 → provider does not support this model. Cache this signal in Redis
+                # (via DualCache) so every router instance skips this deployment next time.
+                # TTL of 1h means we re-test after an hour, in case the provider adds support.
+                if isinstance(e, litellm.NotFoundError):
+                    _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id")
+                    if _dep_id_cap:
+                        asyncio.create_task(
+                            self.cache.async_set_cache(
+                                key=f"litellm:cap:{_dep_id_cap}:{model}",
+                                value=False,
+                                ttl=3600,  # 1h — re-test after TTL
+                            )
+                        )
             raise e
 
     def _update_kwargs_before_fallbacks(
@@ -7330,11 +7354,15 @@ class Router:
         status_code: Final = getattr(error, "status_code", None)
         if status_code is not None and not litellm._should_retry(status_code):
             # 401/403 are special cases - allow retry if multiple deployments exist (handled below)
-            if status_code not in (401, 403):
+            # 404 means the provider does not serve this model, so the retry loop is
+            # allowed to advance to the next-priority deployment. The
+            # `_num_healthy_deployments <= 0` guard at the end of this method still
+            # raises once nothing else is left to try, and the capability cache in
+            # async_get_healthy_deployments keeps the 404-ing deployment out of the
+            # candidate list on subsequent attempts.
+            if status_code not in (401, 403, 404):
                 raise error
 
-        if isinstance(error, litellm.NotFoundError):
-            raise error
         # Error we should only retry if there are other deployments
         if isinstance(error, openai.RateLimitError):
             if (
@@ -11714,21 +11742,46 @@ class Router:
             request_kwargs=request_kwargs,
         )
 
-        ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
-        _target_order: Final = (request_kwargs or {}).pop("_target_order", None)
-        healthy_deployments = litellm.utils._get_order_filtered_deployments(
-            cast(list[dict], healthy_deployments), target_order=_target_order
-        )
+        ## CAPABILITY CACHE FILTER ## -> skip deployments whose 404 responses have been
+        ## cached, indicating they cannot serve this model at this provider. The cache
+        ## is backed by Redis (via self.cache DualCache) with a TTL so providers that
+        ## later add model support are automatically re-tested after expiry.
+        ## key schema: litellm:cap:<deployment_id>:<model_group>
+        ##   None  → unknown (first request) → allow
+        ##   True  → confirmed capable        → allow
+        ##   False → confirmed incapable (404) → skip
+        _capable_deployments: list[dict] = []
+        for _dep in cast(list[dict], healthy_deployments):
+            _dep_id: str | None = (_dep.get("model_info") or {}).get("id")
+            if _dep_id:
+                _cap_key = f"litellm:cap:{_dep_id}:{model}"
+                _cached_cap = await self.cache.async_get_cache(_cap_key)
+                if _cached_cap is not False:  # None or True → keep
+                    _capable_deployments.append(_dep)
+            else:
+                _capable_deployments.append(_dep)  # no id to look up, allow through
+        if _capable_deployments:  # only narrow if filter leaves at least one candidate
+            healthy_deployments = _capable_deployments
 
-        ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in
-        ## this request via weighted-failover. Always honored, regardless of the
-        ## router-level flag, so a stale exclusion key on kwargs cannot escape.
+        ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in this
+        ## request via weighted-failover. Runs BEFORE order filter so that excluded
+        ## order-1 deployments don't prevent order-2 from being selected — this was
+        ## the root cause of silent 404 routing failures with order-based fallback.
         _excluded_deployment_ids: Final = (request_kwargs or {}).pop("_excluded_deployment_ids", None)
         healthy_deployments = litellm.utils._get_excluded_filtered_deployments(
             cast(list[dict], healthy_deployments),
             excluded_deployment_ids=_excluded_deployment_ids,
         )
 
+        ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments
+        ## with the lowest order value (order=1 preferred over order=2). Runs AFTER
+        ## exclusion so incapable/excluded order-1 deployments correctly fall through
+        ## to order-2 rather than producing an empty deployment list.
+        _target_order: Final = (request_kwargs or {}).pop("_target_order", None)
+        healthy_deployments = litellm.utils._get_order_filtered_deployments(
+            cast(list[dict], healthy_deployments), target_order=_target_order
+        )
+
         if len(healthy_deployments) == 0:
             exception: Final = await async_raise_no_deployment_exception(
                 litellm_router_instance=self,

tests/test_litellm/test_router_capability_cache.py

@@ -0,0 +1,212 @@
+"""Tests for the router capability cache and the order/exclusion filter ordering.
+
+Covers three behaviours introduced together:
+
+1. A provider answering 404 for a model no longer aborts the request. The retry
+   loop is allowed to advance to the next-priority deployment.
+2. `async_get_healthy_deployments` applies weighted-failover exclusion BEFORE the
+   `model_info.order` filter, so excluded order-1 deployments fall through to
+   order-2 instead of emptying the candidate list.
+3. A DualCache entry per (deployment id, model group) records which providers
+   answered 404, so later requests skip them until the TTL expires.
+"""
+
+import pytest
+
+import litellm
+from litellm import Router
+
+MODEL_GROUP = "universal-group"
+
+
+def _two_tier_router() -> Router:
+    """Two deployments in one group: order 1 and order 2."""
+    return Router(
+        model_list=[
+            {
+                "model_name": MODEL_GROUP,
+                "litellm_params": {"model": "openai/tier-one", "api_key": "sk-tier-one"},
+                "model_info": {"id": "dep-order-1", "order": 1},
+            },
+            {
+                "model_name": MODEL_GROUP,
+                "litellm_params": {"model": "openai/tier-two", "api_key": "sk-tier-two"},
+                "model_info": {"id": "dep-order-2", "order": 2},
+            },
+        ]
+    )
+
+
+def _flat_router() -> Router:
+    """Two deployments in one group, no order set."""
+    return Router(
+        model_list=[
+            {
+                "model_name": MODEL_GROUP,
+                "litellm_params": {"model": "openai/alpha", "api_key": "sk-alpha"},
+                "model_info": {"id": "dep-alpha"},
+            },
+            {
+                "model_name": MODEL_GROUP,
+                "litellm_params": {"model": "openai/beta", "api_key": "sk-beta"},
+                "model_info": {"id": "dep-beta"},
+            },
+        ]
+    )
+
+
+def _not_found() -> litellm.NotFoundError:
+    return litellm.NotFoundError(
+        message="The model 'some-model' does not exist",
+        model=MODEL_GROUP,
+        llm_provider="openai",
+    )
+
+
+def _ids(deployments) -> set:
+    return {d["model_info"]["id"] for d in deployments}
+
+
+# ---------------------------------------------------------------- retry gating
+
+
+def test_404_retries_while_other_deployments_remain():
+    """A 404 returns True so the retry loop moves to the next deployment."""
+    router = _two_tier_router()
+
+    assert (
+        router.should_retry_this_error(
+            error=_not_found(),
+            healthy_deployments=[{"model_info": {"id": "dep-order-2"}}],
+            all_deployments=router.model_list,
+        )
+        is True
+    )
+
+
+def test_404_raises_once_nothing_healthy_is_left():
+    """The last 404 still surfaces to the caller instead of retrying forever."""
+    router = _two_tier_router()
+
+    with pytest.raises(litellm.NotFoundError):
+        router.should_retry_this_error(
+            error=_not_found(),
+            healthy_deployments=[],
+            all_deployments=router.model_list,
+        )
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        litellm.BadRequestError(message="bad", model=MODEL_GROUP, llm_provider="openai"),
+        litellm.ContextWindowExceededError(
+            message="too long", model=MODEL_GROUP, llm_provider="openai"
+        ),
+    ],
+    ids=["400", "context-window"],
+)
+def test_other_4xx_errors_still_raise(error):
+    """Only 404 was exempted; other non-retryable 4xx keep aborting the request."""
+    router = _two_tier_router()
+
+    with pytest.raises(type(error)):
+        router.should_retry_this_error(
+            error=error,
+            healthy_deployments=[{"model_info": {"id": "dep-order-2"}}],
+            all_deployments=router.model_list,
+        )
+
+
+# ------------------------------------------------------- exclusion before order
+
+
+@pytest.mark.asyncio
+async def test_excluded_order_1_falls_through_to_order_2():
+    """Regression: excluding every order-1 deployment used to empty the list.
+
+    The order filter narrowed to order-1 first, exclusion then removed all of
+    them, and the caller saw "No deployments available" while a healthy order-2
+    deployment was sitting right there.
+    """
+    router = _two_tier_router()
+
+    deployments = await router.async_get_healthy_deployments(
+        model=MODEL_GROUP,
+        request_kwargs={"_excluded_deployment_ids": ["dep-order-1"]},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert _ids(deployments) == {"dep-order-2"}
+
+
+@pytest.mark.asyncio
+async def test_order_1_preferred_when_nothing_is_excluded():
+    """The order filter still wins when exclusion removes nothing."""
+    router = _two_tier_router()
+
+    deployments = await router.async_get_healthy_deployments(
+        model=MODEL_GROUP,
+        request_kwargs={},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert _ids(deployments) == {"dep-order-1"}
+
+
+# --------------------------------------------------------------- capability cache
+
+
+@pytest.mark.asyncio
+async def test_incapable_deployment_is_skipped():
+    """A deployment cached as incapable drops out of the candidate list."""
+    router = _flat_router()
+    await router.cache.async_set_cache(
+        key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=False, ttl=3600
+    )
+
+    deployments = await router.async_get_healthy_deployments(
+        model=MODEL_GROUP,
+        request_kwargs={},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert _ids(deployments) == {"dep-beta"}
+
+
+@pytest.mark.asyncio
+async def test_capable_and_unknown_deployments_are_kept():
+    """True means keep; a missing entry means untested, so also keep."""
+    router = _flat_router()
+    await router.cache.async_set_cache(
+        key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=True, ttl=86400
+    )
+
+    deployments = await router.async_get_healthy_deployments(
+        model=MODEL_GROUP,
+        request_kwargs={},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert _ids(deployments) == {"dep-alpha", "dep-beta"}
+
+
+@pytest.mark.asyncio
+async def test_all_incapable_falls_back_to_the_full_list():
+    """With every candidate cached incapable the filter is skipped.
+
+    Better to re-probe a stale cache than to refuse a request outright.
+    """
+    router = _flat_router()
+    for dep_id in ("dep-alpha", "dep-beta"):
+        await router.cache.async_set_cache(
+            key=f"litellm:cap:{dep_id}:{MODEL_GROUP}", value=False, ttl=3600
+        )
+
+    deployments = await router.async_get_healthy_deployments(
+        model=MODEL_GROUP,
+        request_kwargs={},
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert _ids(deployments) == {"dep-alpha", "dep-beta"}