scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

refactor(router): read distinct deployment orders through a public helper

Browse source 
The buffering gate copied async_function_with_fallbacks' order-scan, so both call
sites reached into litellm.utils._get_deployment_order and the duplicate tripped the
LIT006 and reportPrivateUsage ceilings. Put the scan next to the private accessor as
get_distinct_deployment_orders and call that from both, which drops 4 private reads
where the gate only needed 2 gone.

Also validate the fallback list through a TypeAdapter instead of narrowing it with
cast, and ratchet the basedpyright limits down by what this clears.
This commit is contained in:
nuernber 2026-09-08 12:04:04 -07:00
parent 4049dd17f1
commit 53cacd8e7c
3 changed files with 22 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
basedpyright-code-budget.json
View file

@ -57,7 +57,7 @@
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15281
"limit": 15279
},
"reportMissingTypeStubs": {
"limit": 40
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1804
"limit": 1802
},
"reportRedeclaration": {
"limit": 8
@ -99,7 +99,7 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44358
"limit": 44357
},
"reportUnknownLambdaType": {
"limit": 109
@ -111,7 +111,7 @@
"limit": 19584
},
"reportUnknownVariableType": {
"limit": 29814
"limit": 29812
},
"reportUnnecessaryCast": {
"limit": 110

23
litellm/router.py
View file

@ -403,6 +403,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_FALLBACK_LIST_ADAPTER: Final = TypeAdapter(list[object])
_EXACT_KEY_FALLBACK_ENTRY_ADAPTER: Final = TypeAdapter(dict[str, list[str]])
@ -417,11 +418,11 @@ def _exact_key_fallback_entries(
resolver walks entries one at a time and can return an earlier well-formed entry's chain
without ever reading a malformed one.
"""
if not isinstance(fallbacks, list):
try:
entries: Final = _FALLBACK_LIST_ADAPTER.validate_python(fallbacks)
except ValidationError:
return []
return [
typed for entry in cast(list[object], fallbacks) if (typed := _as_exact_key_fallback_entry(entry)) is not None
]
return [typed for entry in entries if (typed := _as_exact_key_fallback_entry(entry)) is not None]
def _as_exact_key_fallback_entry(entry: object) -> dict[str, list[str]] | None:
@ -7279,12 +7280,7 @@ class Router:
# Use wildcard-aware lookup so order-based fallback also works for model
# groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`).
all_deployments: Final = self.get_model_list(model_name=original_model_group, team_id=_request_team_id) or []
_order_set: Final[set] = {
litellm.utils._get_deployment_order(d)
for d in all_deployments
if litellm.utils._get_deployment_order(d) is not None
}
order_values: Final[list] = sorted(_order_set)
order_values: Final = litellm.utils.get_distinct_deployment_orders(all_deployments)
if len(order_values) > 1 and not _skip_order_fallback:
# Determine which order levels have already been tried
current_target: Final = kwargs.get("_target_order")
@ -8417,12 +8413,7 @@ class Router:
strategy, _ = self._get_routing_context(model_group, kwargs) # pyright: ignore[reportArgumentType] # Mapping is read-only, safe for dict param
if strategy == "simple-shuffle" and len(all_deployments) > 1:
return True
order_values: Final = {
litellm.utils._get_deployment_order(d)
for d in all_deployments
if litellm.utils._get_deployment_order(d) is not None
}
if len(order_values) > 1:
if len(litellm.utils.get_distinct_deployment_orders(all_deployments)) > 1:
return True
lookup_groups: Final = fallback_lookup_groups(kwargs, model_group)
if (

11
litellm/utils.py
View file

@ -4896,6 +4896,17 @@ def _get_deployment_order(deployment: dict | Any) -> int | None:
return order
def get_distinct_deployment_orders(deployments: Sequence[Mapping[str, Any]]) -> tuple[int, ...]:
"""
The ascending distinct `order` levels present across `deployments`, ignoring those without one.
More than one level means the router can retry a failure against a different deployment in the
same model group, so callers deciding whether an order-based retry is reachable read this rather
than each deployment's order.
"""
return tuple(sorted({order for d in deployments for order in [_get_deployment_order(d)] if order is not None}))
def get_order_filtered_deployments(healthy_deployments: list[dict], target_order: int | None = None) -> list:
if target_order is not None:
return [d for d in healthy_deployments if _get_deployment_order(d) == target_order]