Merge 4fc4b59d68 into c5da69cd13

litellm/router_python_core.py

@@ -832,6 +832,10 @@ def _every_deployment_refused(
     return all(_configured_model_info(deployment).get("id") in refused_deployment_ids for deployment in deployments)
 
 
+def _refused_by_no_new_deployment(already_skipped: tuple[str, ...], skipped_now: tuple[str, ...]) -> bool:
+    return frozenset(skipped_now) <= frozenset(already_skipped)
+
+
 def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
     if silent_model is None or in_mirrored_call():
         return ()
@@ -9751,11 +9755,21 @@ class PythonCore(DeprecatedRouterMembers):
 
             ## LOGGING
             if num_retries > 0:
-                kwargs = self._core_log_retry(kwargs=kwargs, e=original_exception)
+                first_already_skipped_ids: Final = as_retry_skipped_deployment_ids(
+                    kwargs.get("_retry_skipped_deployment_ids")
+                )
                 first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry(
                     exception=original_exception,
-                    already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
+                    already_skipped=first_already_skipped_ids,
                 )
+                if isinstance(original_exception, litellm_exceptions.ContextWindowExceededError) and (
+                    _refused_by_no_new_deployment(first_already_skipped_ids, first_skipped_ids)
+                    or _every_deployment_refused(
+                        _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), first_skipped_ids
+                    )
+                ):
+                    raise
+                kwargs = self._core_log_retry(kwargs=kwargs, e=original_exception)
                 if first_skipped_ids:
                     kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids  # rebind-ok: the next attempt reads it
                 if isinstance(
@@ -9840,10 +9854,18 @@ class PythonCore(DeprecatedRouterMembers):
                         except Exception:
                             raise e
 
+                    already_skipped_ids = as_retry_skipped_deployment_ids(kwargs.get("_retry_skipped_deployment_ids"))
                     skipped_ids = self._deployment_ids_to_skip_on_retry(
                         exception=e,
-                        already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
+                        already_skipped=already_skipped_ids,
                     )
+                    if isinstance(e, litellm_exceptions.ContextWindowExceededError) and (
+                        _refused_by_no_new_deployment(already_skipped_ids, skipped_ids)
+                        or _every_deployment_refused(
+                            _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids
+                        )
+                    ):
+                        break
                     if skipped_ids:
                         kwargs["_retry_skipped_deployment_ids"] = skipped_ids  # rebind-ok: the next attempt reads it
                     if isinstance(e, litellm_exceptions.UnsupportedEndpointError) and _every_deployment_refused(
@@ -9964,7 +9986,9 @@ class PythonCore(DeprecatedRouterMembers):
         status_code: Final = getattr(error, "status_code", None)
         if status_code is not None and not self._host.invoke("should_retry")(status_code):
             # 401/403 are special cases - allow retry if multiple deployments exist (handled below)
-            if status_code not in (401, 403) and not isinstance(error, litellm_exceptions.UnsupportedEndpointError):
+            if status_code not in (401, 403) and not isinstance(
+                error, (litellm_exceptions.UnsupportedEndpointError, litellm_exceptions.ContextWindowExceededError)
+            ):
                 raise error
 
         if isinstance(error, litellm_exceptions.NotFoundError):

tests/unit/test_router/test_router.py

@@ -22710,6 +22710,173 @@ _LIT_7114_BATCH_OK = {
     "status": "validating",
     "created_at": 1,
 }
+_UPSTREAM_CONTEXT_WINDOW_400 = {
+    "message": "This model's maximum context length is 16385 tokens",
+    "type": "invalid_request_error",
+    "code": "context_length_exceeded",
+}
+
+
+@pytest.mark.asyncio
+async def test_router_retries_a_context_window_refusal_on_a_sibling_deployment(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0])
+    router = litellm.Router(
+        model_list=[_retry_skip_deployment("refusing", "refusing"), _retry_skip_deployment("sibling", "sibling")],
+        num_retries=2,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        refusing = respx_mock.post("https://refusing.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        sibling = respx_mock.post("https://sibling.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert refusing.call_count == 1
+    assert sibling.call_count == 1
+    assert response.choices[0].message.content == "hi back"
+
+
+@pytest.mark.asyncio
+async def test_router_tries_each_deployment_once_when_every_one_refuses_the_context_window(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0])
+    router = litellm.Router(
+        model_list=[_retry_skip_deployment("first", "first"), _retry_skip_deployment("second", "second")],
+        num_retries=3,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        first = respx_mock.post("https://first.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        second = respx_mock.post("https://second.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        with pytest.raises(litellm.ContextWindowExceededError):
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert first.call_count == 1
+    assert second.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_router_retries_a_context_window_refusal_at_most_once_when_no_sibling_fits(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router = litellm.Router(
+        model_list=[
+            _retry_skip_deployment("admitting", "admitting", model_info={"max_input_tokens": 1_000_000}),
+            _retry_skip_deployment("too-small", "too-small", model_info={"max_input_tokens": 1}),
+        ],
+        num_retries=3,
+        enable_pre_call_checks=True,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        admitting = respx_mock.post("https://admitting.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        too_small = respx_mock.post("https://too-small.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        with pytest.raises(litellm.ContextWindowExceededError) as raised:
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert "maximum context length" in str(raised.value)
+    assert too_small.call_count == 0
+    assert admitting.call_count <= 2
+    assert raised.value.num_retries == admitting.call_count - 1
+
+
+@pytest.mark.asyncio
+async def test_router_raises_its_own_context_window_error_without_retrying_when_no_deployment_fits(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router = litellm.Router(
+        model_list=[
+            _retry_skip_deployment("small", "small", model_info={"max_input_tokens": 1}),
+            _retry_skip_deployment("smaller", "smaller", model_info={"max_input_tokens": 1}),
+        ],
+        num_retries=2,
+        enable_pre_call_checks=True,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        small = respx_mock.post("https://small.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        smaller = respx_mock.post("https://smaller.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        with pytest.raises(litellm.ContextWindowExceededError) as raised:
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert small.call_count == smaller.call_count == 0
+    assert not raised.value.num_retries
+
+
+@pytest.mark.asyncio
+async def test_router_makes_one_call_for_a_context_window_refusal_in_a_one_deployment_group(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2, disable_cooldowns=True)
+
+    with respx.mock as respx_mock:
+        only = respx_mock.post("https://only.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        with pytest.raises(litellm.ContextWindowExceededError):
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert only.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_router_sends_a_context_window_refusal_to_context_window_fallbacks_without_retrying_the_group(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0])
+    router = litellm.Router(
+        model_list=[
+            _retry_skip_deployment("refusing", "refusing"),
+            _retry_skip_deployment("sibling", "sibling"),
+            {**_retry_skip_deployment("long", "long"), "model_name": "gpt-5.6-long"},
+        ],
+        num_retries=2,
+        context_window_fallbacks=[{"gpt-5.6": ["gpt-5.6-long"]}],
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        refusing = respx_mock.post("https://refusing.local/v1/chat/completions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_CONTEXT_WINDOW_400})
+        )
+        sibling = respx_mock.post("https://sibling.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        long = respx_mock.post("https://long.local/v1/chat/completions").mock(
+            return_value=httpx.Response(200, json=_LIT_7114_CHAT_OK)
+        )
+        response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert refusing.call_count == 1
+    assert sibling.call_count == 0
+    assert long.call_count == 1
+    assert response.choices[0].message.content == "hi back"
 
 
 class _PassthroughAdapter(CustomLogger):