mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(router): keep retry skips on the active order-fallback target
The retry-skip guard checks that some other deployment could still answer before it excludes the one that just refused, so a single-deployment group keeps the old retry-in-place behavior. It asked that question at the group's minimum order, but the router picks the retry's deployment at the order the request has already escalated to. So a group with a primary at order 1 and a backup at order 2 answered "yes, order 1 still has a candidate" while the retry was pinned to order 2, and the exclusion left order 2 with nothing. The caller got a no-deployments error in place of the provider's own 400. The helper now takes the active target order and filters by it, which is the same value async_get_healthy_deployments reads off the request.
This commit is contained in:
parent
7c7810df42
commit
cb1ec76e46
2 changed files with 37 additions and 1 deletions
|
|
@ -402,6 +402,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
|
|||
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
|
||||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_EXCLUDED_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
_TARGET_ORDER_ADAPTER: Final = TypeAdapter(int | None)
|
||||
|
||||
|
||||
def _with_router_resolved_session_model(session: object, model_name: str) -> Mapping[str, Mapping[str, object]]:
|
||||
|
|
@ -7464,6 +7465,7 @@ class Router:
|
|||
exception: Exception,
|
||||
already_skipped: object,
|
||||
healthy_deployments: list[dict], # mutable-ok: matches the routing filters' list contract
|
||||
target_order: object = None,
|
||||
) -> tuple[str, ...]:
|
||||
failed_deployment_id: Final[str | None] = getattr(exception, "failed_deployment_id", None)
|
||||
status_code: Final = getattr(exception, "status_code", None)
|
||||
|
|
@ -7473,7 +7475,9 @@ class Router:
|
|||
return ()
|
||||
already_skipped_ids: Final = _EXCLUDED_DEPLOYMENT_IDS_ADAPTER.validate_python(already_skipped or ())
|
||||
skipped: Final = frozenset((*already_skipped_ids, failed_deployment_id))
|
||||
same_order_candidates: Final = litellm.utils.get_order_filtered_deployments(healthy_deployments)
|
||||
same_order_candidates: Final = litellm.utils.get_order_filtered_deployments(
|
||||
healthy_deployments, target_order=_TARGET_ORDER_ADAPTER.validate_python(target_order)
|
||||
)
|
||||
if not litellm.utils.get_excluded_filtered_deployments(same_order_candidates, excluded_deployment_ids=skipped):
|
||||
return ()
|
||||
verbose_router_logger.debug(
|
||||
|
|
@ -7580,6 +7584,7 @@ class Router:
|
|||
exception=original_exception,
|
||||
already_skipped=kwargs.get("_excluded_deployment_ids"),
|
||||
healthy_deployments=_healthy_deployments,
|
||||
target_order=kwargs.get("_target_order"),
|
||||
)
|
||||
if skipped_deployment_ids:
|
||||
kwargs["_excluded_deployment_ids"] = skipped_deployment_ids
|
||||
|
|
@ -7656,6 +7661,7 @@ class Router:
|
|||
exception=e,
|
||||
already_skipped=kwargs.get("_excluded_deployment_ids"),
|
||||
healthy_deployments=_healthy_deployments,
|
||||
target_order=kwargs.get("_target_order"),
|
||||
)
|
||||
if retry_skipped_deployment_ids:
|
||||
kwargs["_excluded_deployment_ids"] = retry_skipped_deployment_ids
|
||||
|
|
|
|||
|
|
@ -13261,6 +13261,36 @@ def test_router_deployment_ids_to_skip_on_retry(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target_order,deployment_orders,expected",
|
||||
[
|
||||
(2, {"rejecting": 2, "sibling": 1}, ()),
|
||||
(2, {"rejecting": 2, "sibling": 2}, ("rejecting",)),
|
||||
(1, {"rejecting": 1, "sibling": 2}, ()),
|
||||
(None, {"rejecting": 1, "sibling": 2}, ()),
|
||||
(None, {"rejecting": 1, "sibling": 1}, ("rejecting",)),
|
||||
(3, {"rejecting": 2, "sibling": 1}, ()),
|
||||
],
|
||||
)
|
||||
def test_router_deployment_ids_to_skip_on_retry_honors_order_fallback_target(
|
||||
target_order, deployment_orders, expected
|
||||
):
|
||||
exception = Exception("upstream refused this request")
|
||||
exception.status_code = 400
|
||||
exception.failed_deployment_id = "rejecting"
|
||||
healthy_deployments = [
|
||||
{"model_info": {"id": deployment_id}, "litellm_params": {"order": order}}
|
||||
for deployment_id, order in deployment_orders.items()
|
||||
]
|
||||
|
||||
assert (
|
||||
litellm.Router._deployment_ids_to_skip_on_retry(
|
||||
exception, None, healthy_deployments, target_order=target_order
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def _make_failure_logging_obj():
|
||||
return LiteLLMLogging(
|
||||
model="gpt-5.6",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue