From 040c7b53d31cc9c11b5d02c0b43a4f5c88cf0f80 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 26 Mar 2026 11:54:32 +0530 Subject: [PATCH] feat(router): add order-based fallback so higher order deployments are tried on failure When order=1 deployments fail, the router now automatically tries order=2, then order=3, etc. before falling through to external fallbacks. This removes the need for enable_pre_call_checks and makes order work as a true priority-based fallback chain within a model group. Co-Authored-By: Claude Opus 4.6 --- litellm/router.py | 57 +++ .../test_router_order_fallback.py | 330 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 tests/test_litellm/test_router_order_fallback.py diff --git a/litellm/router.py b/litellm/router.py index b64a4d1e599..1dde9c9a6eb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5288,6 +5288,63 @@ class Router: if "fallback_depth" not in input_kwargs: input_kwargs["fallback_depth"] = 0 + # ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list + # Skip for error types that have their own dedicated fallback handlers + _skip_order_fallback = isinstance( + e, + (litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError), + ) + all_deployments = self._get_all_deployments(model_name=original_model_group) + _order_set: set = { + d.get("litellm_params", {}).get("order") + for d in all_deployments + if d.get("litellm_params", {}).get("order") is not None + } + order_values: list = sorted(_order_set) + if len(order_values) > 1 and not _skip_order_fallback: + # Determine which order levels have already been tried + current_target = kwargs.get("_target_order") + skip_up_to = ( + current_target if current_target is not None else order_values[0] + ) + # Build order-based fallback entries (skip already-tried levels) + order_fallback_entries: List = [ + {"model": original_model_group, "_target_order": o} + for o in order_values + if o > skip_up_to + ] + # Get external fallbacks — handle both standard and non-standard formats + external_fallback_group: Optional[List] = None + if fallbacks is not None and model_group is not None: + if _check_non_standard_fallback_format(fallbacks=fallbacks): + # Non-standard formats (e.g. ["claude-3-haiku"] or + # [{"model": "...", "messages": [...]}]) are passed through directly + external_fallback_group = fallbacks + else: + external_fallback_group, generic_idx = get_fallback_model_group( + fallbacks=fallbacks, + model_group=cast(str, model_group), + ) + if external_fallback_group is None and generic_idx is not None: + external_fallback_group = fallbacks[generic_idx]["*"] + # Combined list: order fallbacks first, then external + combined_fallbacks = order_fallback_entries + ( + external_fallback_group or [] + ) + + if combined_fallbacks: + input_kwargs.update( + { + "fallback_model_group": combined_fallbacks, + "original_model_group": original_model_group, + } + ) + response = await run_async_fallback( + *args, + **input_kwargs, + ) + return response + try: verbose_router_logger.info("Trying to fallback b/w models") diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py new file mode 100644 index 00000000000..6ea0a863eba --- /dev/null +++ b/tests/test_litellm/test_router_order_fallback.py @@ -0,0 +1,330 @@ +""" +Tests for order-based fallback routing. + +When deployments have `order` set in litellm_params, lower order deployments +should be tried first, and higher order deployments should be used as fallbacks +when lower order deployments fail. +""" + +from typing import Optional + +import pytest + +from litellm import Router +from litellm.utils import _get_order_filtered_deployments + +# --------------------------------------------------------------------------- +# Unit tests for _get_order_filtered_deployments +# --------------------------------------------------------------------------- + + +class TestGetOrderFilteredDeployments: + def _make_deployment(self, order: Optional[int], dep_id: str) -> dict: + params: dict = {"model": "gpt-4o", "api_key": "key"} + if order is not None: + params["order"] = order + return { + "model_name": "test-model", + "litellm_params": params, + "model_info": {"id": dep_id}, + } + + def test_returns_min_order_group(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + self._make_deployment(1, "c"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + assert all(d["model_info"]["id"] in ("a", "c") for d in result) + + def test_target_order_filters_to_exact_level(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + self._make_deployment(3, "c"), + ] + result = _get_order_filtered_deployments(deps, target_order=2) + assert len(result) == 1 + assert result[0]["model_info"]["id"] == "b" + + def test_target_order_no_match_returns_all(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(2, "b"), + ] + result = _get_order_filtered_deployments(deps, target_order=99) + assert len(result) == 2 + + def test_no_order_set_returns_all(self): + deps = [ + self._make_deployment(None, "a"), + self._make_deployment(None, "b"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + + def test_empty_list(self): + result = _get_order_filtered_deployments([]) + assert result == [] + + def test_single_order_returns_all_with_that_order(self): + deps = [ + self._make_deployment(1, "a"), + self._make_deployment(1, "b"), + ] + result = _get_order_filtered_deployments(deps) + assert len(result) == 2 + + +# --------------------------------------------------------------------------- +# Integration tests for order-based fallback in Router +# --------------------------------------------------------------------------- + + +def test_router_order_without_pre_call_checks(): + """Order filtering should work even when enable_pre_call_checks=False (default).""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 1", + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + enable_pre_call_checks=False, + ) + + for _ in range(20): + response = router.completion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "1" + + +def test_router_order_no_fallback_when_healthy(): + """When order=1 is healthy, order=2 should never be used.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 1", + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + ) + + for _ in range(50): + response = router.completion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "1" + + +@pytest.mark.asyncio +async def test_router_order_fallback_on_failure(): + """When order=1 fails, order=2 should be tried as fallback.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad-key", + "mock_response": Exception("connection error"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good-key", + "mock_response": "success from order 2", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "2" + + +@pytest.mark.asyncio +async def test_router_order_fallback_three_levels(): + """When order=1 and order=2 both fail, order=3 should be tried.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from order 3", + "order": 3, + }, + "model_info": {"id": "3"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "3" + + +@pytest.mark.asyncio +async def test_router_order_fallback_then_external_fallback(): + """When all order levels fail, external fallbacks should be tried.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from external fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + fallbacks=[{"test-model": ["fallback-model"]}], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + assert response._hidden_params["model_id"] == "fallback" + +@pytest.mark.asyncio +async def test_router_order_fallback_with_non_standard_fallbacks(): + """Non-standard fallback formats (e.g. fallbacks=["model-name"]) passed + per-request should still be tried after all order levels are exhausted.""" + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("fail order 2"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "success from non-standard fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + num_retries=0, + ) + + response = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + fallbacks=["fallback-model"], # non-standard format, passed per-request + ) + assert response._hidden_params["model_id"] == "fallback" \ No newline at end of file