From b2eda10befe8402daaa914d30ff3f49d5fd7a506 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 19 Oct 2026 17:52:05 -0700 Subject: [PATCH] perf(router): resolve deployment usage limits once per deployment instead of every request (#47393) _increment_deployment_usage looked up the deployment, resolved its model info and model_dump'ed it on every call just to decide whether the deployment has any tpm/rpm/itpm/otpm limit. That answer only changes when the model list or discovered limits change, so it is now memoized per deployment and cleared in _invalidate_model_group_info_cache --- litellm/router.py | 53 ++++++++------- .../test_router/test_router_helper_utils.py | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+), 23 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5bf46c68b20..d8da69f7789 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1389,6 +1389,9 @@ class Router: ) self._ptu_share_teams = lru_cache(maxsize=1)(self._collect_ptu_share_teams) self._cached_team_ptu_ceiling = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(self._resolve_team_ptu_ceiling) + self._cached_deployment_tracks_usage = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._deployment_tracks_usage + ) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -9053,31 +9056,10 @@ class Router: ) -> str | None: from litellm.types.caching import RedisPipelineIncrementOperation - deployment_info: Final = self.get_deployment(model_id=deployment_id) - if deployment_info is None: - return None - deployment_model_info: Final = self.get_router_model_info( - deployment=deployment_info, - received_model_name=model_group, - ) - configured_limits: Final = ( - deployment_info.get("tpm", None), - deployment_info.get("rpm", None), - deployment_info.litellm_params.tpm, - deployment_info.litellm_params.rpm, - deployment_model_info.get("tpm", None), - deployment_model_info.get("rpm", None), - ) - ## Nothing to track only when neither tpm/rpm nor itpm/otpm limits are - ## set. IO deployments still record TPM/RPM usage here so TPM-aware - ## routing strategies see their real load in mixed model groups; their - ## itpm/otpm enforcement runs separately in ModelRateLimitingCheck. - if all(limit is None for limit in configured_limits) and not deployment_has_io_token_limits( - deployment_info.model_dump() - ): - return None if total_tokens <= 0 and rpm_increment <= 0: return None + if not self._cached_deployment_tracks_usage(deployment_id, model_group): + return None current_minute: Final = get_utc_datetime().strftime("%H-%M") # use the same timezone regardless of system clock tpm_key: Final = RouterCacheEnum.TPM.value.format( @@ -9101,6 +9083,30 @@ class Router: ) return tpm_key + def _deployment_tracks_usage(self, deployment_id: str, model_group: str) -> bool: + deployment_info: Final = self.get_deployment(model_id=deployment_id) + if deployment_info is None: + return False + deployment_model_info: Final = self.get_router_model_info( + deployment=deployment_info, + received_model_name=model_group, + ) + configured_limits: Final = ( + deployment_info.get("tpm", None), + deployment_info.get("rpm", None), + deployment_info.litellm_params.tpm, + deployment_info.litellm_params.rpm, + deployment_model_info.get("tpm", None), + deployment_model_info.get("rpm", None), + ) + ## Nothing to track only when neither tpm/rpm nor itpm/otpm limits are + ## set. IO deployments still record TPM/RPM usage here so TPM-aware + ## routing strategies see their real load in mixed model groups; their + ## itpm/otpm enforcement runs separately in ModelRateLimitingCheck. + return any(limit is not None for limit in configured_limits) or deployment_has_io_token_limits( + deployment_info.model_dump() + ) + def sync_deployment_callback_on_success( self, kwargs, # kwargs to completion @@ -13386,6 +13392,7 @@ class Router: self.cached_deployment_model_info.cache_clear() self._ptu_share_teams.cache_clear() self._cached_team_ptu_ceiling.cache_clear() + self._cached_deployment_tracks_usage.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index ce60cce8408..e3061294bef 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1,3 +1,4 @@ +import functools import json import os from datetime import datetime, timezone @@ -1276,6 +1277,69 @@ async def test_increment_deployment_usage_writes_only_positive_deltas_for_limite assert await unlimited.get_model_group_usage("gpt-5-mini") == (None, None) +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_increment_deployment_usage_starts_tracking_after_limit_is_added(): + router = Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake"}, + "model_info": {"id": "lit-3058-upsert"}, + } + ] + ) + increment = functools.partial( + router._increment_deployment_usage, + deployment_id="lit-3058-upsert", + deployment_name="gpt-5-mini", + model_group="gpt-5-mini", + total_tokens=25, + rpm_increment=1, + parent_otel_span=None, + ) + + assert await increment() is None + + router.upsert_deployment( + Deployment( + model_name="gpt-5-mini", + litellm_params=LiteLLM_Params(model="gpt-5-mini", api_key="sk-fake", rpm=10), + model_info={"id": "lit-3058-upsert"}, + ) + ) + + assert await increment() is not None + assert await router.get_model_group_usage("gpt-5-mini") == (25, 1) + + +@pytest.mark.parametrize( + "litellm_params, model_info, deployment_id, tracks", + [ + ({}, {}, "lit-3058-tracks", False), + ({"rpm": 10}, {}, "lit-3058-tracks", True), + ({"tpm": 1000}, {}, "lit-3058-tracks", True), + ({"otpm": 500}, {}, "lit-3058-tracks", True), + ({}, {"itpm": 500}, "lit-3058-tracks", True), + ({"rpm": 10}, {}, "lit-3058-unknown", False), + ], +) +def test_deployment_tracks_usage_for_every_limit_source( + litellm_params: dict, model_info: dict, deployment_id: str, tracks: bool +): + router = Router( + model_list=[ + { + "model_name": "my-unmapped-model", + "litellm_params": {"model": "openai/my-unmapped-model", "api_key": "sk-fake", **litellm_params}, + "model_info": {"id": "lit-3058-tracks", **model_info}, + } + ] + ) + + assert router._deployment_tracks_usage(deployment_id, "my-unmapped-model") is tracks + + def _shared_redis_stub(store: dict) -> MagicMock: from litellm.caching.redis_cache import RedisCache