From bab7ea4c22a5f2df410e09c053b6d02e69c35eb4 Mon Sep 17 00:00:00 2001 From: nuernber Date: Thu, 3 Sep 2026 16:45:12 -0700 Subject: [PATCH] fix(router): respect disable_fallbacks and check weighted failover viability Addresses two PR review comments: 1. When disable_fallbacks=True is set on a streaming Anthropic Messages request, skip the buffer-until-content path even if fallbacks are configured. The request explicitly opted out of recovery, so withholding lifecycle frames provides no benefit and only adds latency. 2. When enable_weighted_failover is enabled, only report a recovery path if the routing strategy is simple-shuffle AND there are multiple deployments available. Weighted failover cannot select an alternative deployment for single-deployment groups or non-simple-shuffle strategies, so the unconditional return was incorrectly triggering buffering with no actual fallback protection. Both issues caused Anthropic lifecycle frames (message_start, content_block_start) to be buffered until visible content arrived, preserving the adaptive-thinking delay this PR is intended to remove. --- litellm/router.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8c677fc7d63..a748d17f63e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5166,7 +5166,9 @@ class Router: source_iterator: Final = response model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group - if not self._has_any_configured_fallback(model_group, initial_kwargs): + if fallbacks_disabled_for_request(initial_kwargs) or not self._has_any_configured_fallback( + model_group, initial_kwargs + ): # Nothing to fall back to, so buffering lifecycle frames to protect a mid-stream # fallback attempt would only add latency for no benefit: forward the source # iterator live, exactly as it would stream without this wrapper. @@ -8174,10 +8176,12 @@ class Router: fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) if _check_non_standard_fallback_format(fallbacks=fallbacks): return True - if self.enable_weighted_failover: - return True team_id: Final = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") all_deployments: Final = self.get_model_list(model_name=model_group, team_id=team_id) or [] + if self.enable_weighted_failover: + strategy, _ = self._get_routing_context(model_group, kwargs) # pyright: ignore[reportArgumentType] # Mapping is read-only, safe for dict param + if strategy == "simple-shuffle" and len(all_deployments) > 1: + return True order_values: Final = { litellm.utils._get_deployment_order(d) for d in all_deployments