From 53cacd8e7c37dea4a776491f380eec400bede30f Mon Sep 17 00:00:00 2001 From: nuernber Date: Tue, 8 Sep 2026 12:04:04 -0700 Subject: [PATCH] refactor(router): read distinct deployment orders through a public helper The buffering gate copied async_function_with_fallbacks' order-scan, so both call sites reached into litellm.utils._get_deployment_order and the duplicate tripped the LIT006 and reportPrivateUsage ceilings. Put the scan next to the private accessor as get_distinct_deployment_orders and call that from both, which drops 4 private reads where the gate only needed 2 gone. Also validate the fallback list through a TypeAdapter instead of narrowing it with cast, and ratchet the basedpyright limits down by what this clears. --- basedpyright-code-budget.json | 8 ++++---- litellm/router.py | 23 +++++++---------------- litellm/utils.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 57ca267e504..107b521a1bc 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -57,7 +57,7 @@ "limit": 5570 }, "reportMissingTypeArgument": { - "limit": 15281 + "limit": 15279 }, "reportMissingTypeStubs": { "limit": 40 @@ -84,7 +84,7 @@ "limit": 56 }, "reportPrivateUsage": { - "limit": 1804 + "limit": 1802 }, "reportRedeclaration": { "limit": 8 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44358 + "limit": 44357 }, "reportUnknownLambdaType": { "limit": 109 @@ -111,7 +111,7 @@ "limit": 19584 }, "reportUnknownVariableType": { - "limit": 29814 + "limit": 29812 }, "reportUnnecessaryCast": { "limit": 110 diff --git a/litellm/router.py b/litellm/router.py index 4ebf1bc871e..375a6b0f96d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -403,6 +403,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_FALLBACK_LIST_ADAPTER: Final = TypeAdapter(list[object]) _EXACT_KEY_FALLBACK_ENTRY_ADAPTER: Final = TypeAdapter(dict[str, list[str]]) @@ -417,11 +418,11 @@ def _exact_key_fallback_entries( resolver walks entries one at a time and can return an earlier well-formed entry's chain without ever reading a malformed one. """ - if not isinstance(fallbacks, list): + try: + entries: Final = _FALLBACK_LIST_ADAPTER.validate_python(fallbacks) + except ValidationError: return [] - return [ - typed for entry in cast(list[object], fallbacks) if (typed := _as_exact_key_fallback_entry(entry)) is not None - ] + return [typed for entry in entries if (typed := _as_exact_key_fallback_entry(entry)) is not None] def _as_exact_key_fallback_entry(entry: object) -> dict[str, list[str]] | None: @@ -7279,12 +7280,7 @@ class Router: # Use wildcard-aware lookup so order-based fallback also works for model # groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`). all_deployments: Final = self.get_model_list(model_name=original_model_group, team_id=_request_team_id) or [] - _order_set: Final[set] = { - litellm.utils._get_deployment_order(d) - for d in all_deployments - if litellm.utils._get_deployment_order(d) is not None - } - order_values: Final[list] = sorted(_order_set) + order_values: Final = litellm.utils.get_distinct_deployment_orders(all_deployments) if len(order_values) > 1 and not _skip_order_fallback: # Determine which order levels have already been tried current_target: Final = kwargs.get("_target_order") @@ -8417,12 +8413,7 @@ class Router: strategy, _ = self._get_routing_context(model_group, kwargs) # pyright: ignore[reportArgumentType] # Mapping is read-only, safe for dict param if strategy == "simple-shuffle" and len(all_deployments) > 1: return True - order_values: Final = { - litellm.utils._get_deployment_order(d) - for d in all_deployments - if litellm.utils._get_deployment_order(d) is not None - } - if len(order_values) > 1: + if len(litellm.utils.get_distinct_deployment_orders(all_deployments)) > 1: return True lookup_groups: Final = fallback_lookup_groups(kwargs, model_group) if ( diff --git a/litellm/utils.py b/litellm/utils.py index d0e11bc9551..d95dce6910b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4896,6 +4896,17 @@ def _get_deployment_order(deployment: dict | Any) -> int | None: return order +def get_distinct_deployment_orders(deployments: Sequence[Mapping[str, Any]]) -> tuple[int, ...]: + """ + The ascending distinct `order` levels present across `deployments`, ignoring those without one. + + More than one level means the router can retry a failure against a different deployment in the + same model group, so callers deciding whether an order-based retry is reachable read this rather + than each deployment's order. + """ + return tuple(sorted({order for d in deployments for order in [_get_deployment_order(d)] if order is not None})) + + def get_order_filtered_deployments(healthy_deployments: list[dict], target_order: int | None = None) -> list: if target_order is not None: return [d for d in healthy_deployments if _get_deployment_order(d) == target_order]