diff --git a/litellm/router.py b/litellm/router.py index 5ab7623aa3e..8c677fc7d63 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8164,10 +8164,27 @@ class Router: every model group unconditionally rather than being keyed by one at all - self._get_fallback_model_group_for_lookup_groups checks neither, so using it here would report "nothing to fall back to" for a request that a real error would in fact retry. + + Two more retry paths in the same dispatcher fire without any of `fallbacks` / + `context_window_fallbacks` / `content_policy_fallbacks` configured at all: order-based + fallback (deployments in the model group at more than one `order` level) and weighted + intra-group failover (`enable_weighted_failover`), both of which pick a different + deployment for the retry, not the one that already streamed lifecycle frames live. """ fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) if _check_non_standard_fallback_format(fallbacks=fallbacks): return True + if self.enable_weighted_failover: + return True + team_id: Final = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") + all_deployments: Final = self.get_model_list(model_name=model_group, team_id=team_id) or [] + order_values: Final = { + litellm.utils._get_deployment_order(d) + for d in all_deployments + if litellm.utils._get_deployment_order(d) is not None + } + if len(order_values) > 1: + return True lookup_groups: Final = fallback_lookup_groups(kwargs, model_group) candidate_fallback_lists: Final = ( fallbacks, diff --git a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py index 4eb4b8d9260..1e10c186571 100644 --- a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py +++ b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py @@ -544,6 +544,37 @@ def test_has_any_configured_fallback_matches_non_standard_client_fallbacks(): assert router._has_any_configured_fallback("fable-tier", {"fallbacks": ["opus-target"]}) is True +def test_has_any_configured_fallback_arms_on_order_based_deployments(): + """Regression: async_function_with_fallbacks_common_utils retries against a higher-order + deployment in the same model group whenever more than one `order` level is present, even + with zero `fallbacks`/`context_window_fallbacks`/`content_policy_fallbacks` configured - + the gate must recognize that retry path too, or a mid-stream error can still trigger an + order-based retry that appends a second message_start onto a stream already forwarded live.""" + router = Router( + model_list=[ + { + "model_name": "fable-tier", + "litellm_params": {"model": "anthropic/claude-fable-5", "api_key": "sk-test", "order": 1}, + }, + { + "model_name": "fable-tier", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "sk-test", "order": 2}, + }, + ] + ) + + assert router._has_any_configured_fallback("fable-tier", {}) is True + + +def test_has_any_configured_fallback_arms_on_weighted_failover(): + """Regression: enable_weighted_failover lets a retryable failure re-pick across the + model group's other deployments before any cross-group fallback runs, independent of + `fallbacks` config entirely - the gate must arm for it too.""" + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], enable_weighted_failover=True) + + assert router._has_any_configured_fallback("fable-tier", {}) is True + + def test_get_fallback_model_group_for_lookup_groups_orders_tier_before_requested(): router = _router(content_policy_fallbacks=None) fallbacks = [{"tier1": ["backup-a"]}, {"smart-router": ["backup-b"]}]