fix(router)!: scope context fallback precedence to the requested group (#49779)

* fix(router): scope context fallback precedence to the requested group

* test(router): keep existing fallback test formatting

* fix(router): preserve context errors after ordered backup failures

litellm/router_python_core.py

@@ -9152,11 +9152,17 @@ class PythonCore(DeprecatedRouterMembers):
         if include_fallback_errors:
             input_kwargs["include_fallback_errors"] = True
 
-        # ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list
-        # Skip for error types that have their own dedicated fallback handlers
-        _skip_order_fallback: Final = isinstance(
-            e,
-            (litellm_exceptions.ContextWindowExceededError, litellm_exceptions.ContentPolicyViolationError),
+        context_window_fallback_model_group: Final = (
+            self._get_fallback_model_group_for_lookup_groups(
+                fallbacks=context_window_fallbacks,
+                lookup_groups=lookup_groups,
+            )
+            if isinstance(e, litellm_exceptions.ContextWindowExceededError) and context_window_fallbacks is not None
+            else None
+        )
+        _skip_order_fallback: Final = (
+            isinstance(e, litellm_exceptions.ContentPolicyViolationError)
+            or context_window_fallback_model_group is not None
         )
         _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id")
         # Use wildcard-aware lookup so order-based fallback also works for model
@@ -9202,14 +9208,23 @@ class PythonCore(DeprecatedRouterMembers):
                         "original_model_group": original_model_group,
                     }
                 )
-                response = await run_async_fallback(
-                    *args,
-                    **input_kwargs,
-                )
-                return response
+                try:
+                    return await run_async_fallback(
+                        *args,
+                        **input_kwargs,
+                    )
+                except Exception:
+                    if isinstance(original_exception, litellm_exceptions.ContextWindowExceededError):
+                        raise original_exception
+                    raise
 
         # Weighted intra-group failover (simple-shuffle only); see _maybe_run_weighted_failover.
-        if self.enable_weighted_failover and not _skip_order_fallback and original_model_group is not None:
+        if (
+            self.enable_weighted_failover
+            and not _skip_order_fallback
+            and not isinstance(e, litellm_exceptions.ContextWindowExceededError)
+            and original_model_group is not None
+        ):
             response = await self._maybe_run_weighted_failover(
                 exception=e,
                 original_model_group=original_model_group,
@@ -9227,7 +9242,7 @@ class PythonCore(DeprecatedRouterMembers):
             # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks=[{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey, how's it going?"}]}]
             is_non_standard_fallback_format: Final = check_non_standard_fallback_format(fallbacks=fallbacks)
 
-            if is_non_standard_fallback_format:
+            if is_non_standard_fallback_format and context_window_fallback_model_group is None:
                 input_kwargs.update(
                     {
                         "fallback_model_group": fallbacks,
@@ -9243,16 +9258,7 @@ class PythonCore(DeprecatedRouterMembers):
                 return response
 
             if isinstance(e, litellm_exceptions.ContextWindowExceededError):
-                if context_window_fallbacks is not None:
-                    context_window_fallback_model_group: Final[list[str] | None] = (
-                        self._get_fallback_model_group_for_lookup_groups(
-                            fallbacks=context_window_fallbacks,
-                            lookup_groups=lookup_groups,
-                        )
-                    )
-                    if context_window_fallback_model_group is None:
-                        raise original_exception
-
+                if context_window_fallback_model_group is not None:
                     input_kwargs.update(
                         {
                             "fallback_model_group": context_window_fallback_model_group,
@@ -9266,7 +9272,7 @@ class PythonCore(DeprecatedRouterMembers):
                     )
                     return response
 
-                else:
+                elif context_window_fallbacks is None:
                     error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
                     verbose_router_logger.info(
                         msg=f"Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \

tests/unit/test_router/core/test_fallbacks.py

@@ -1,6 +1,7 @@
 """Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised."""
 
 from collections.abc import Callable, Mapping
+from typing import Final
 
 import pytest
 
@@ -355,21 +356,13 @@ async def test_an_error_specific_chain_wins_over_the_general_chain(
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
-    ("error", "field", "raised"),
-    [
-        (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError),
-        (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError),
-    ],
-    ids=["context-window", "content-policy"],
-)
-async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain(
-    make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception]
+async def test_an_unmatched_content_policy_chain_does_not_try_the_general_chain(
+    make_core: MakeCore, host: ScriptedHost
 ) -> None:
-    core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]})
-    host.answer("openai/a", error())
+    core: Final = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}])
+    host.answer("openai/a", content_policy())
 
-    with pytest.raises(raised):
+    with pytest.raises(litellm.ContentPolicyViolationError):
         await complete(core)
 
     assert host.deployments_called("acompletion") == ["a"]

tests/unit/test_router/test_router_fallbacks.py

@@ -1207,3 +1207,156 @@ def test_context_stream_fallback_preserves_other_errors_for_existing_continuatio
         message="provider unavailable", model="primary", llm_provider="openai", generated_content="already visible"
     )
     assert _context_window_stream_fallback_error(error, "primary", "openai", True) is error
+
+
+def _context_fallback_precedence_router(context_key: str | None, ordered: bool) -> Router:
+    deployments: Final = (
+        ("primary", "primary", 1),
+        ("primary" if ordered else "unused", "ordered", 2),
+        ("generic", "generic", 1),
+        ("context", "context", 1),
+    )
+    return Router(
+        model_list=[
+            {
+                "model_name": group,
+                "litellm_params": {
+                    "model": f"openai/{model}",
+                    "api_key": "test-key",
+                    "api_base": "https://fallback-migration.local/v1",
+                    "order": order,
+                },
+            }
+            for group, model, order in deployments
+        ],
+        fallbacks=[{"primary": ["generic"]}],
+        context_window_fallbacks=None if context_key is None else [{context_key: ["context"]}],
+        num_retries=0,
+        disable_cooldowns=True,
+    )
+
+
+@pytest.mark.parametrize("sync_mode", (False, True))
+@pytest.mark.parametrize(
+    ("context_key", "ordered", "flat_fallbacks", "expected_model"),
+    (
+        (None, True, False, "ordered"),
+        ("unrelated", True, False, "ordered"),
+        ("unrelated", False, False, "generic"),
+        ("primary", True, False, "context"),
+        ("*", True, False, "context"),
+        ("primary", True, True, "context"),
+        ("primary", False, True, "context"),
+    ),
+)
+@pytest.mark.asyncio
+async def test_context_fallback_precedence_is_specific_to_the_requested_group(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    sync_mode: bool,
+    context_key: str | None,
+    ordered: bool,
+    flat_fallbacks: bool,
+    expected_model: str,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router: Final = _context_fallback_precedence_router(context_key, ordered)
+    route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock(
+        side_effect=(
+            httpx.Response(
+                400,
+                json={
+                    "error": {
+                        "message": "This model's maximum context length is 10 tokens",
+                        "type": "invalid_request_error",
+                        "code": "context_length_exceeded",
+                    }
+                },
+            ),
+            _fallback_chat_response(expected_model, f"served by {expected_model}"),
+        )
+    )
+
+    response: Final = (
+        router.completion(
+            model="primary",
+            messages=[{"role": "user", "content": "long prompt"}],
+            fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}],
+        )
+        if sync_mode
+        else await router.acompletion(
+            model="primary",
+            messages=[{"role": "user", "content": "long prompt"}],
+            fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}],
+        )
+    )
+
+    assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", expected_model)
+    assert response.choices[0].message.content == f"served by {expected_model}"
+
+
+@pytest.mark.parametrize("matching_context", (False, True))
+@pytest.mark.asyncio
+async def test_context_fallback_exhaustion_preserves_the_selected_chain(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    matching_context: bool,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router: Final = _context_fallback_precedence_router("primary" if matching_context else "unrelated", True)
+    refusal: Final = httpx.Response(
+        400,
+        json={"error": {"message": "This model's maximum context length is 10 tokens"}},
+    )
+    route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock(
+        side_effect=(refusal, refusal, _fallback_chat_response("generic", "generic answer"))
+    )
+
+    if matching_context:
+        with pytest.raises(litellm.ContextWindowExceededError):
+            await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}])
+        assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "context")
+        return
+
+    response: Final = await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}])
+    assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic")
+    assert response.choices[0].message.content == "generic answer"
+
+
+@pytest.mark.parametrize("sync_mode", (False, True))
+@pytest.mark.parametrize("context_error", (False, True))
+@pytest.mark.parametrize("backup_status", (429, 503))
+@pytest.mark.asyncio
+async def test_ordered_fallback_failures_preserve_the_original_context_error(
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+    sync_mode: bool,
+    context_error: bool,
+    backup_status: int,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router: Final = _context_fallback_precedence_router("unrelated", True)
+    original_message: Final = (
+        "This model's maximum context length is 10 tokens" if context_error else "primary unavailable"
+    )
+    backup_error: Final = litellm.RateLimitError if backup_status == 429 else litellm.ServiceUnavailableError
+    expected_error: Final = litellm.ContextWindowExceededError if context_error else backup_error
+    route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock(
+        side_effect=(
+            httpx.Response(400 if context_error else 503, json={"error": {"message": original_message}}),
+            httpx.Response(backup_status, json={"error": {"message": "ordered deployment unavailable"}}),
+            httpx.Response(backup_status, json={"error": {"message": "generic deployment unavailable"}}),
+        )
+    )
+
+    request: Final = (
+        asyncio.to_thread(router.completion, model="primary", messages=[{"role": "user", "content": "long prompt"}])
+        if sync_mode
+        else router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}])
+    )
+    with pytest.raises(expected_error) as raised:
+        await request
+
+    assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic")
+    assert raised.value.status_code == (400 if context_error else backup_status)
+    assert (original_message if context_error else "generic deployment unavailable") in str(raised.value)