From 13b734046015c2db07f5ec8932b60e210261169f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:41:35 -0700 Subject: [PATCH] fix(router): retry a context-window error on a sibling deployment that fits --- litellm/router.py | 32 ++++- tests/unit/test_router/test_router.py | 167 ++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b63130c3313..fc4f14d6e0b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -725,6 +725,10 @@ def _every_deployment_refused( return all(_configured_model_info(deployment).get("id") in refused_deployment_ids for deployment in deployments) +def _refused_by_no_new_deployment(already_skipped: tuple[str, ...], skipped_now: tuple[str, ...]) -> bool: + return frozenset(skipped_now) <= frozenset(already_skipped) + + def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: if silent_model is None: return () @@ -9164,11 +9168,21 @@ class Router: ## LOGGING if num_retries > 0: - kwargs = self.log_retry(kwargs=kwargs, e=original_exception) + first_already_skipped_ids: Final = as_retry_skipped_deployment_ids( + kwargs.get("_retry_skipped_deployment_ids") + ) first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( exception=original_exception, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + already_skipped=first_already_skipped_ids, ) + if isinstance(original_exception, litellm.ContextWindowExceededError) and ( + _refused_by_no_new_deployment(first_already_skipped_ids, first_skipped_ids) + or _every_deployment_refused( + _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), first_skipped_ids + ) + ): + raise + kwargs = self.log_retry(kwargs=kwargs, e=original_exception) if first_skipped_ids: kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it if isinstance(original_exception, litellm.UnsupportedEndpointError) and _every_deployment_refused( @@ -9248,10 +9262,18 @@ class Router: except Exception: raise e + already_skipped_ids = as_retry_skipped_deployment_ids(kwargs.get("_retry_skipped_deployment_ids")) skipped_ids = self._deployment_ids_to_skip_on_retry( exception=e, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + already_skipped=already_skipped_ids, ) + if isinstance(e, litellm.ContextWindowExceededError) and ( + _refused_by_no_new_deployment(already_skipped_ids, skipped_ids) + or _every_deployment_refused( + _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids + ) + ): + break if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused( @@ -9354,7 +9376,9 @@ class Router: status_code: Final = getattr(error, "status_code", None) if status_code is not None and not litellm.should_retry(status_code): # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403) and not isinstance(error, litellm.UnsupportedEndpointError): + if status_code not in (401, 403) and not isinstance( + error, (litellm.UnsupportedEndpointError, litellm.ContextWindowExceededError) + ): raise error if isinstance(error, litellm.NotFoundError): diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 300c9cff6ff..34100f2837e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -21928,6 +21928,173 @@ _LIT_7114_BATCH_OK = { "status": "validating", "created_at": 1, } +_UPSTREAM_CONTEXT_WINDOW_400 = { + "message": "This model's maximum context length is 16385 tokens", + "type": "invalid_request_error", + "code": "context_length_exceeded", +} + + +@pytest.mark.asyncio +async def test_router_retries_a_context_window_refusal_on_a_sibling_deployment(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0]) + router = litellm.Router( + model_list=[_retry_skip_deployment("refusing", "refusing"), _retry_skip_deployment("sibling", "sibling")], + num_retries=2, + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + refusing = respx_mock.post("https://refusing.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + sibling = respx_mock.post("https://sibling.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert refusing.call_count == 1 + assert sibling.call_count == 1 + assert response.choices[0].message.content == "hi back" + + +@pytest.mark.asyncio +async def test_router_tries_each_deployment_once_when_every_one_refuses_the_context_window( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0]) + router = litellm.Router( + model_list=[_retry_skip_deployment("first", "first"), _retry_skip_deployment("second", "second")], + num_retries=3, + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + first = respx_mock.post("https://first.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + second = respx_mock.post("https://second.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + with pytest.raises(litellm.ContextWindowExceededError): + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert first.call_count == 1 + assert second.call_count == 1 + + +@pytest.mark.asyncio +async def test_router_retries_a_context_window_refusal_at_most_once_when_no_sibling_fits( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("admitting", "admitting", model_info={"max_input_tokens": 1_000_000}), + _retry_skip_deployment("too-small", "too-small", model_info={"max_input_tokens": 1}), + ], + num_retries=3, + enable_pre_call_checks=True, + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + admitting = respx_mock.post("https://admitting.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + too_small = respx_mock.post("https://too-small.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert "maximum context length" in str(raised.value) + assert too_small.call_count == 0 + assert admitting.call_count <= 2 + assert raised.value.num_retries == admitting.call_count - 1 + + +@pytest.mark.asyncio +async def test_router_raises_its_own_context_window_error_without_retrying_when_no_deployment_fits( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("small", "small", model_info={"max_input_tokens": 1}), + _retry_skip_deployment("smaller", "smaller", model_info={"max_input_tokens": 1}), + ], + num_retries=2, + enable_pre_call_checks=True, + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + small = respx_mock.post("https://small.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + smaller = respx_mock.post("https://smaller.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + with pytest.raises(litellm.ContextWindowExceededError) as raised: + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert small.call_count == smaller.call_count == 0 + assert not raised.value.num_retries + + +@pytest.mark.asyncio +async def test_router_makes_one_call_for_a_context_window_refusal_in_a_one_deployment_group( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2, disable_cooldowns=True) + + with respx.mock as respx_mock: + only = respx_mock.post("https://only.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + with pytest.raises(litellm.ContextWindowExceededError): + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert only.call_count == 1 + + +@pytest.mark.asyncio +async def test_router_sends_a_context_window_refusal_to_context_window_fallbacks_without_retrying_the_group( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0]) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("refusing", "refusing"), + _retry_skip_deployment("sibling", "sibling"), + {**_retry_skip_deployment("long", "long"), "model_name": "gpt-5.6-long"}, + ], + num_retries=2, + context_window_fallbacks=[{"gpt-5.6": ["gpt-5.6-long"]}], + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + refusing = respx_mock.post("https://refusing.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400}) + ) + sibling = respx_mock.post("https://sibling.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + long = respx_mock.post("https://long.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert refusing.call_count == 1 + assert sibling.call_count == 0 + assert long.call_count == 1 + assert response.choices[0].message.content == "hi back" class _PassthroughAdapter(CustomLogger):