diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index ebba764eb8c..b7e0a58523c 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -12,6 +12,7 @@ is logged the first time such a deployment is seen. import contextlib from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -47,6 +48,9 @@ class RoutingArgs: ttl: int = 60 # 1min (RPM/TPM expire key) +_NO_LITELLM_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({}) + + class ModelRateLimitingCheck(CustomLogger): """ Pre-call check that enforces TPM/RPM or ITPM/OTPM limits on model deployments. @@ -314,7 +318,7 @@ class ModelRateLimitingCheck(CustomLogger): ``model_info`` in, so ``model_id`` comes back as "". The router does stamp ``kwargs["model_info"]``, so prefer that before giving up. """ - litellm_params: Final = kwargs.get("litellm_params") or {} + litellm_params: Final = kwargs.get("litellm_params") or _NO_LITELLM_PARAMS for source in (kwargs.get("model_info"), litellm_params.get("model_info")): if isinstance(source, Mapping): candidate = source.get("id")