fix(router): keep retry skips on the active order-fallback target

The retry-skip guard checks that some other deployment could still answer
before it excludes the one that just refused, so a single-deployment group
keeps the old retry-in-place behavior. It asked that question at the group's
minimum order, but the router picks the retry's deployment at the order the
request has already escalated to.

So a group with a primary at order 1 and a backup at order 2 answered "yes,
order 1 still has a candidate" while the retry was pinned to order 2, and the
exclusion left order 2 with nothing. The caller got a no-deployments error in
place of the provider's own 400.

The helper now takes the active target order and filters by it, which is the
same value async_get_healthy_deployments reads off the request.
This commit is contained in:
mateo-berri 2026-09-05 23:02:44 -07:00
parent 7c7810df42
commit cb1ec76e46
2 changed files with 37 additions and 1 deletions

View file

@ -402,6 +402,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_EXCLUDED_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
_TARGET_ORDER_ADAPTER: Final = TypeAdapter(int | None)
def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]:
@ -7464,6 +7465,7 @@ class Router:
exception: Exception,
already_skipped: object,
healthy_deployments: list[dict], # mutable-ok: matches the routing filters' list contract
target_order: object = None,
) -> tuple[str, ...]:
failed_deployment_id: Final[str | None] = getattr(exception, "failed_deployment_id", None)
status_code: Final = getattr(exception, "status_code", None)
@ -7473,7 +7475,9 @@ class Router:
return ()
already_skipped_ids: Final = _EXCLUDED_DEPLOYMENT_IDS_ADAPTER.validate_python(already_skipped or ())
skipped: Final = frozenset((*already_skipped_ids, failed_deployment_id))
same_order_candidates: Final = litellm.utils.get_order_filtered_deployments(healthy_deployments)
same_order_candidates: Final = litellm.utils.get_order_filtered_deployments(
healthy_deployments, target_order=_TARGET_ORDER_ADAPTER.validate_python(target_order)
)
if not litellm.utils.get_excluded_filtered_deployments(same_order_candidates, excluded_deployment_ids=skipped):
return ()
verbose_router_logger.debug(
@ -7580,6 +7584,7 @@ class Router:
exception=original_exception,
already_skipped=kwargs.get("_excluded_deployment_ids"),
healthy_deployments=_healthy_deployments,
target_order=kwargs.get("_target_order"),
)
if skipped_deployment_ids:
kwargs["_excluded_deployment_ids"] = skipped_deployment_ids
@ -7656,6 +7661,7 @@ class Router:
exception=e,
already_skipped=kwargs.get("_excluded_deployment_ids"),
healthy_deployments=_healthy_deployments,
target_order=kwargs.get("_target_order"),
)
if retry_skipped_deployment_ids:
kwargs["_excluded_deployment_ids"] = retry_skipped_deployment_ids

View file

@ -13261,6 +13261,36 @@ def test_router_deployment_ids_to_skip_on_retry(
)
@pytest.mark.parametrize(
"target_order,deployment_orders,expected",
[
(2, {"rejecting": 2, "sibling": 1}, ()),
(2, {"rejecting": 2, "sibling": 2}, ("rejecting",)),
(1, {"rejecting": 1, "sibling": 2}, ()),
(None, {"rejecting": 1, "sibling": 2}, ()),
(None, {"rejecting": 1, "sibling": 1}, ("rejecting",)),
(3, {"rejecting": 2, "sibling": 1}, ()),
],
)
def test_router_deployment_ids_to_skip_on_retry_honors_order_fallback_target(
target_order, deployment_orders, expected
):
exception = Exception("upstream refused this request")
exception.status_code = 400
exception.failed_deployment_id = "rejecting"
healthy_deployments = [
{"model_info": {"id": deployment_id}, "litellm_params": {"order": order}}
for deployment_id, order in deployment_orders.items()
]
assert (
litellm.Router._deployment_ids_to_skip_on_retry(
exception, None, healthy_deployments, target_order=target_order
)
== expected
)
def _make_failure_logging_obj():
return LiteLLMLogging(
model="gpt-5.6",