From ef1cde433ea7c6dd1515de06c6d0d748fae4a197 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:22:14 -0700 Subject: [PATCH 1/3] fix: add moonshot/kimi-k3 to the cost map models.litellm.ai and released litellm versions read model_prices_and_context_window.json from main at runtime, so Kimi K3 is missing from the hosted catalog even though the entry is in review for litellm_internal_staging in #37552. This copies that entry onto main so the catalog picks it up on its next fetch. Data only: the cost map and its backup copy, no code changes. Pricing matches Moonshot's published rates ($3/M input, $0.30/M cache read, $15/M output, 1,048,576-token context). The fireworks_ai and Azure Foundry kimi-k3 variants are separate work in #37512 and #37658; neither touches the native moonshot/kimi-k3 key. --- .../model_prices_and_context_window_backup.json | 17 +++++++++++++++++ model_prices_and_context_window.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 07f9027313b..53d069c4a71 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30025,6 +30025,23 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://platform.kimi.ai/docs/pricing/chat-k3", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, "deprecation_date": "2026-01-28", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 07f9027313b..53d069c4a71 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30025,6 +30025,23 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://platform.kimi.ai/docs/pricing/chat-k3", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, "deprecation_date": "2026-01-28", From ed5ce943d83c75a4d081bb6d7fdae986c8c9a5a0 Mon Sep 17 00:00:00 2001 From: Vadym Yehorov Date: Mon, 24 Aug 2026 22:16:08 -0400 Subject: [PATCH 2/3] feat(router): add Redis-backed capability cache + fix order/exclusion filter order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem ------- 1. When a provider returns HTTP 404 (model not found), litellm immediately raises NotFoundError in should_retry_this_error(), preventing the retry loop from advancing to the next deployment. 2. async_get_healthy_deployments() runs ORDER filtering BEFORE EXCLUSION filtering. Root cause: if all order-1 deployments are excluded (already tried), the order filter narrows to order-1 only, then exclusion empties the list → NoDeploymentException instead of falling through to order-2. Changes ------- 1. should_retry_this_error: remove the immediate NotFoundError raise; let the retry loop continue so order-based fallback can advance to the next provider. 2. async_get_healthy_deployments: a) Add CAPABILITY CACHE FILTER (Redis-backed, TTL-aware) that skips deployments whose 404s have been cached. Key: litellm:cap::. None/True → keep; False → skip. Only narrows if at least 1 capable dep remains. b) Move EXCLUSION filter to run BEFORE ORDER filter (root-cause fix). 3. _acompletion success path: fire-and-forget cache write of True (24h TTL) so all router instances know this deployment can serve this model. 4. _acompletion except block: on NotFoundError, fire-and-forget cache write of False (1h TTL) so subsequent requests skip this deployment until re-tested. Net effect: zero hardcoded model lists, zero custom_callbacks, self-learning via Redis with TTL-based expiry for automatic re-testing. --- litellm/router.py | 72 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 045fd32847c..9b3059a5fd6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3059,6 +3059,17 @@ class Router: ): await response.fetch_stream() + # Record that this deployment successfully served this model. + # Stored in Redis (via DualCache) so all router instances share the signal. + _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id") if deployment else None + if _dep_id_cap: + asyncio.create_task( + self.cache.async_set_cache( + key=f"litellm:cap:{_dep_id_cap}:{model}", + value=True, + ttl=86400, # 24h — re-confirm capability daily + ) + ) self.success_calls[model_name] += 1 verbose_router_logger.info("litellm.acompletion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) # debug how often this deployment picked @@ -3096,6 +3107,19 @@ class Router: if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) self._stamp_failed_deployment_id_with_effective_model_info(e, deployment, kwargs) + # 404 → provider does not support this model. Cache this signal in Redis + # (via DualCache) so every router instance skips this deployment next time. + # TTL of 1h means we re-test after an hour, in case the provider adds support. + if isinstance(e, litellm.NotFoundError): + _dep_id_cap: str | None = (deployment.get("model_info") or {}).get("id") + if _dep_id_cap: + asyncio.create_task( + self.cache.async_set_cache( + key=f"litellm:cap:{_dep_id_cap}:{model}", + value=False, + ttl=3600, # 1h — re-test after TTL + ) + ) raise e def _update_kwargs_before_fallbacks( @@ -6908,8 +6932,11 @@ class Router: if status_code not in (401, 403): raise error - if isinstance(error, litellm.NotFoundError): - raise error + # HTTP 404 (litellm.NotFoundError) means the provider does not support + # this model. We allow the retry loop to continue so that order-based + # deployment fallback can advance to the next-priority provider. + # The capability cache in async_get_healthy_deployments ensures the + # incapable deployment is filtered out on the next attempt. # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): if ( @@ -11280,21 +11307,46 @@ class Router: request_kwargs=request_kwargs, ) - ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).pop("_target_order", None) - healthy_deployments = litellm.utils._get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order - ) + ## CAPABILITY CACHE FILTER ## -> skip deployments whose 404 responses have been + ## cached, indicating they cannot serve this model at this provider. The cache + ## is backed by Redis (via self.cache DualCache) with a TTL so providers that + ## later add model support are automatically re-tested after expiry. + ## key schema: litellm:cap:: + ## None → unknown (first request) → allow + ## True → confirmed capable → allow + ## False → confirmed incapable (404) → skip + _capable_deployments: list[dict] = [] + for _dep in cast(list[dict], healthy_deployments): + _dep_id: str | None = (_dep.get("model_info") or {}).get("id") + if _dep_id: + _cap_key = f"litellm:cap:{_dep_id}:{model}" + _cached_cap = await self.cache.async_get_cache(_cap_key) + if _cached_cap is not False: # None or True → keep + _capable_deployments.append(_dep) + else: + _capable_deployments.append(_dep) # no id to look up, allow through + if _capable_deployments: # only narrow if filter leaves at least one candidate + healthy_deployments = _capable_deployments - ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in - ## this request via weighted-failover. Always honored, regardless of the - ## router-level flag, so a stale exclusion key on kwargs cannot escape. + ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in this + ## request via weighted-failover. Runs BEFORE order filter so that excluded + ## order-1 deployments don't prevent order-2 from being selected — this was + ## the root cause of silent 404 routing failures with order-based fallback. _excluded_deployment_ids: Final = (request_kwargs or {}).pop("_excluded_deployment_ids", None) healthy_deployments = litellm.utils._get_excluded_filtered_deployments( cast(list[dict], healthy_deployments), excluded_deployment_ids=_excluded_deployment_ids, ) + ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments + ## with the lowest order value (order=1 preferred over order=2). Runs AFTER + ## exclusion so incapable/excluded order-1 deployments correctly fall through + ## to order-2 rather than producing an empty deployment list. + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) + healthy_deployments = litellm.utils._get_order_filtered_deployments( + cast(list[dict], healthy_deployments), target_order=_target_order + ) + if len(healthy_deployments) == 0: exception: Final = await async_raise_no_deployment_exception( litellm_router_instance=self, From 940f57b3b8945f79afc725fd984170716c584d78 Mon Sep 17 00:00:00 2001 From: Vadym Yehorov Date: Wed, 26 Aug 2026 00:45:05 -0400 Subject: [PATCH 3/3] test(router): cover capability cache + fix 404 retry guard The earlier commit removed the `isinstance(error, litellm.NotFoundError)` raise in should_retry_this_error, but that block was dead code: the generic guard above it already raises for any status code `litellm._should_retry` rejects, and _should_retry(404) is False. A 404 still aborted the request. Exempt 404 alongside 401/403 in that guard. The `_num_healthy_deployments <= 0` check at the end of the method still raises once nothing is left to try, so a genuinely missing model surfaces to the caller after one attempt per order tier. Adds tests/test_litellm/test_router_capability_cache.py (9 tests) covering: * 404 retries while other deployments remain, raises when none do * 400 and context-window errors still abort, so only 404 was exempted * excluded order-1 deployments fall through to order-2 * order-1 still preferred when exclusion removes nothing * cached-incapable deployments are skipped * capable and untested deployments are kept * every-candidate-incapable falls back to the full list At the merge base 3 of the 9 fail; on this commit all 9 pass. --- litellm/router.py | 13 +- .../test_router_capability_cache.py | 212 ++++++++++++++++++ 2 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/test_router_capability_cache.py diff --git a/litellm/router.py b/litellm/router.py index 9b3059a5fd6..635039afc5d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6929,14 +6929,15 @@ class Router: status_code: Final = getattr(error, "status_code", None) if status_code is not None and not litellm._should_retry(status_code): # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403): + # 404 means the provider does not serve this model, so the retry loop is + # allowed to advance to the next-priority deployment. The + # `_num_healthy_deployments <= 0` guard at the end of this method still + # raises once nothing else is left to try, and the capability cache in + # async_get_healthy_deployments keeps the 404-ing deployment out of the + # candidate list on subsequent attempts. + if status_code not in (401, 403, 404): raise error - # HTTP 404 (litellm.NotFoundError) means the provider does not support - # this model. We allow the retry loop to continue so that order-based - # deployment fallback can advance to the next-priority provider. - # The capability cache in async_get_healthy_deployments ensures the - # incapable deployment is filtered out on the next attempt. # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): if ( diff --git a/tests/test_litellm/test_router_capability_cache.py b/tests/test_litellm/test_router_capability_cache.py new file mode 100644 index 00000000000..91a2717a095 --- /dev/null +++ b/tests/test_litellm/test_router_capability_cache.py @@ -0,0 +1,212 @@ +"""Tests for the router capability cache and the order/exclusion filter ordering. + +Covers three behaviours introduced together: + +1. A provider answering 404 for a model no longer aborts the request. The retry + loop is allowed to advance to the next-priority deployment. +2. `async_get_healthy_deployments` applies weighted-failover exclusion BEFORE the + `model_info.order` filter, so excluded order-1 deployments fall through to + order-2 instead of emptying the candidate list. +3. A DualCache entry per (deployment id, model group) records which providers + answered 404, so later requests skip them until the TTL expires. +""" + +import pytest + +import litellm +from litellm import Router + +MODEL_GROUP = "universal-group" + + +def _two_tier_router() -> Router: + """Two deployments in one group: order 1 and order 2.""" + return Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/tier-one", "api_key": "sk-tier-one"}, + "model_info": {"id": "dep-order-1", "order": 1}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/tier-two", "api_key": "sk-tier-two"}, + "model_info": {"id": "dep-order-2", "order": 2}, + }, + ] + ) + + +def _flat_router() -> Router: + """Two deployments in one group, no order set.""" + return Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/alpha", "api_key": "sk-alpha"}, + "model_info": {"id": "dep-alpha"}, + }, + { + "model_name": MODEL_GROUP, + "litellm_params": {"model": "openai/beta", "api_key": "sk-beta"}, + "model_info": {"id": "dep-beta"}, + }, + ] + ) + + +def _not_found() -> litellm.NotFoundError: + return litellm.NotFoundError( + message="The model 'some-model' does not exist", + model=MODEL_GROUP, + llm_provider="openai", + ) + + +def _ids(deployments) -> set: + return {d["model_info"]["id"] for d in deployments} + + +# ---------------------------------------------------------------- retry gating + + +def test_404_retries_while_other_deployments_remain(): + """A 404 returns True so the retry loop moves to the next deployment.""" + router = _two_tier_router() + + assert ( + router.should_retry_this_error( + error=_not_found(), + healthy_deployments=[{"model_info": {"id": "dep-order-2"}}], + all_deployments=router.model_list, + ) + is True + ) + + +def test_404_raises_once_nothing_healthy_is_left(): + """The last 404 still surfaces to the caller instead of retrying forever.""" + router = _two_tier_router() + + with pytest.raises(litellm.NotFoundError): + router.should_retry_this_error( + error=_not_found(), + healthy_deployments=[], + all_deployments=router.model_list, + ) + + +@pytest.mark.parametrize( + "error", + [ + litellm.BadRequestError(message="bad", model=MODEL_GROUP, llm_provider="openai"), + litellm.ContextWindowExceededError( + message="too long", model=MODEL_GROUP, llm_provider="openai" + ), + ], + ids=["400", "context-window"], +) +def test_other_4xx_errors_still_raise(error): + """Only 404 was exempted; other non-retryable 4xx keep aborting the request.""" + router = _two_tier_router() + + with pytest.raises(type(error)): + router.should_retry_this_error( + error=error, + healthy_deployments=[{"model_info": {"id": "dep-order-2"}}], + all_deployments=router.model_list, + ) + + +# ------------------------------------------------------- exclusion before order + + +@pytest.mark.asyncio +async def test_excluded_order_1_falls_through_to_order_2(): + """Regression: excluding every order-1 deployment used to empty the list. + + The order filter narrowed to order-1 first, exclusion then removed all of + them, and the caller saw "No deployments available" while a healthy order-2 + deployment was sitting right there. + """ + router = _two_tier_router() + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={"_excluded_deployment_ids": ["dep-order-1"]}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-order-2"} + + +@pytest.mark.asyncio +async def test_order_1_preferred_when_nothing_is_excluded(): + """The order filter still wins when exclusion removes nothing.""" + router = _two_tier_router() + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-order-1"} + + +# --------------------------------------------------------------- capability cache + + +@pytest.mark.asyncio +async def test_incapable_deployment_is_skipped(): + """A deployment cached as incapable drops out of the candidate list.""" + router = _flat_router() + await router.cache.async_set_cache( + key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=False, ttl=3600 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-beta"} + + +@pytest.mark.asyncio +async def test_capable_and_unknown_deployments_are_kept(): + """True means keep; a missing entry means untested, so also keep.""" + router = _flat_router() + await router.cache.async_set_cache( + key=f"litellm:cap:dep-alpha:{MODEL_GROUP}", value=True, ttl=86400 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-alpha", "dep-beta"} + + +@pytest.mark.asyncio +async def test_all_incapable_falls_back_to_the_full_list(): + """With every candidate cached incapable the filter is skipped. + + Better to re-probe a stale cache than to refuse a request outright. + """ + router = _flat_router() + for dep_id in ("dep-alpha", "dep-beta"): + await router.cache.async_set_cache( + key=f"litellm:cap:{dep_id}:{MODEL_GROUP}", value=False, ttl=3600 + ) + + deployments = await router.async_get_healthy_deployments( + model=MODEL_GROUP, + request_kwargs={}, + messages=[{"role": "user", "content": "hi"}], + ) + + assert _ids(deployments) == {"dep-alpha", "dep-beta"}