From 9c87124d22267fd56bd27fcc70d690e111da1646 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Sun, 19 Jul 2026 00:16:44 +0100 Subject: [PATCH 1/7] fix(router): skip RPM cache reads when no deployment configures rpm Router._pre_call_checks() read the RPM cache on every request via self.cache.get_cache(), plus a second per-deployment cache read inside the loop, even when no deployment in the model group configures `rpm` - pure overhead on the hot path for the common case of RPM-less deployments. Guards both reads behind a static check of whether any deployment in the group has litellm_params.rpm set, mirroring the existing usage-based-routing-v2 exclusion. Safe since rpm is a direct litellm_params setting, unlike max_input_tokens, which get_router_model_info() can also derive from the cost map and therefore can't be checked the same way. --- litellm/router.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac0933e158b..bfb7ab1561b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14077,13 +14077,21 @@ class Router: else frozenset() ) + # Safe to check statically since `rpm` is a direct litellm_params setting, unlike + # max_input_tokens which get_router_model_info() can also derive from the cost map. + _any_deployment_has_rpm = self.routing_strategy != "usage-based-routing-v2" and any( + (d.get("litellm_params") or {}).get("rpm") is not None for d in _returned_deployments + ) + ## get model group RPM ## - dt: Final = get_utc_datetime() - current_minute: Final = dt.strftime("%H-%M") - rpm_key: Final = f"{model}:rpm:{current_minute}" - model_group_cache: Final = ( - self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {} - ) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache. + model_group_cache: dict[str, int] | None = None + if _any_deployment_has_rpm: + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + rpm_key = f"{model}:rpm:{current_minute}" + model_group_cache = ( + self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {} + ) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache. for idx, deployment in enumerate(_returned_deployments): # Cache nested dict access to avoid repeated temporary dict allocations _litellm_params = deployment.get("litellm_params", {}) @@ -14106,14 +14114,14 @@ class Router: ) continue - model_id = _model_info.get("id", "") ## RPM CHECK ## - ### get local router cache ### - current_request_cache_local = ( - self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0 - ) - ### get usage based cache ### - if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2": + if _any_deployment_has_rpm and isinstance(model_group_cache, dict): + model_id = _model_info.get("id", "") + ### get local router cache ### + current_request_cache_local = ( + self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0 + ) + ### get usage based cache ### model_group_cache[model_id] = model_group_cache.get(model_id, 0) current_request = max(current_request_cache_local, model_group_cache[model_id]) From 698d05dfff0634b737b81d8f7629ea52798766ff Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Tue, 28 Jul 2026 20:26:04 +0100 Subject: [PATCH 2/7] fix: suppress LIT001 on mutated model_group_cache dict annotation --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index bfb7ab1561b..bf5f32f9c7b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14084,7 +14084,7 @@ class Router: ) ## get model group RPM ## - model_group_cache: dict[str, int] | None = None + model_group_cache: dict[str, int] | None = None # mutable-ok: incremented below if _any_deployment_has_rpm: dt = get_utc_datetime() current_minute = dt.strftime("%H-%M") From 0e582644c4a99caa90d75c0ef449b7bef5900262 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 30 Sep 2026 11:44:22 +0200 Subject: [PATCH 3/7] fix: satisfy type discipline gate for rpm cache guard --- litellm/router.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index bf5f32f9c7b..6cbad19dbb3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14080,7 +14080,8 @@ class Router: # Safe to check statically since `rpm` is a direct litellm_params setting, unlike # max_input_tokens which get_router_model_info() can also derive from the cost map. _any_deployment_has_rpm = self.routing_strategy != "usage-based-routing-v2" and any( - (d.get("litellm_params") or {}).get("rpm") is not None for d in _returned_deployments + isinstance(litellm_params := d.get("litellm_params"), dict) and litellm_params.get("rpm") is not None + for d in _returned_deployments ) ## get model group RPM ## @@ -14089,9 +14090,10 @@ class Router: dt = get_utc_datetime() current_minute = dt.strftime("%H-%M") rpm_key = f"{model}:rpm:{current_minute}" - model_group_cache = ( - self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {} - ) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache. + cached_model_group: Final = self.cache.get_cache( + key=rpm_key, local_only=True, parent_otel_span=parent_otel_span + ) + model_group_cache = cached_model_group or {} # mutable-ok: updated below for idx, deployment in enumerate(_returned_deployments): # Cache nested dict access to avoid repeated temporary dict allocations _litellm_params = deployment.get("litellm_params", {}) From 01e81369bba799fe0b28c1c6ca6ff3eacdc538d4 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 14 Oct 2026 12:21:29 +0100 Subject: [PATCH 4/7] fix(router): keep the rpm cache skip inside the type rules The old pre-call test file was deleted upstream, so the regression coverage now sits with the other router pre-call tests. Co-authored-by: Cursor --- litellm/router.py | 20 ++-- tests/unit/test_router/test_router.py | 152 ++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 11 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 6cbad19dbb3..d7a3e54fac0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14079,21 +14079,19 @@ class Router: # Safe to check statically since `rpm` is a direct litellm_params setting, unlike # max_input_tokens which get_router_model_info() can also derive from the cost map. - _any_deployment_has_rpm = self.routing_strategy != "usage-based-routing-v2" and any( + any_deployment_has_rpm: Final = self.routing_strategy != "usage-based-routing-v2" and any( isinstance(litellm_params := d.get("litellm_params"), dict) and litellm_params.get("rpm") is not None for d in _returned_deployments ) ## get model group RPM ## - model_group_cache: dict[str, int] | None = None # mutable-ok: incremented below - if _any_deployment_has_rpm: - dt = get_utc_datetime() - current_minute = dt.strftime("%H-%M") - rpm_key = f"{model}:rpm:{current_minute}" - cached_model_group: Final = self.cache.get_cache( - key=rpm_key, local_only=True, parent_otel_span=parent_otel_span - ) - model_group_cache = cached_model_group or {} # mutable-ok: updated below + rpm_key: Final = f"{model}:rpm:{get_utc_datetime().strftime('%H-%M')}" + cached_model_group: Final = ( + self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) + if any_deployment_has_rpm + else None + ) + model_group_cache: Final = cached_model_group if isinstance(cached_model_group, dict) else {} for idx, deployment in enumerate(_returned_deployments): # Cache nested dict access to avoid repeated temporary dict allocations _litellm_params = deployment.get("litellm_params", {}) @@ -14117,7 +14115,7 @@ class Router: continue ## RPM CHECK ## - if _any_deployment_has_rpm and isinstance(model_group_cache, dict): + if any_deployment_has_rpm: model_id = _model_info.get("id", "") ### get local router cache ### current_request_cache_local = ( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4a9dd199c08..40d60214b4c 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -24748,6 +24748,158 @@ class TestPreCallChecksOptimization: assert deployments[0].get("model_info", {}).get("id") == "small", "First deployment ID changed!" assert deployments[1].get("model_info", {}).get("id") == "large", "Second deployment ID changed!" + +def _rpm_cache_keys_read_by(router: Router, model: str, deployments: list) -> list: + get_cache_calls: list = [] + original_get_cache = router.cache.get_cache + + def _tracking_get_cache(*args, **kwargs): + get_cache_calls.append(kwargs.get("key", args[0] if args else None)) + return original_get_cache(*args, **kwargs) + + router.cache.get_cache = _tracking_get_cache + router._pre_call_checks( + model=model, + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + return get_cache_calls + + +def test_no_rpm_cache_read_when_no_deployment_has_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm-1"}, + }, + { + "model_name": "test", + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm-2"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments) + + assert get_cache_calls == [], f"Expected zero cache reads, got {get_cache_calls}" + + +def test_rpm_cache_still_read_and_enforced_when_a_deployment_has_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1}, + "model_info": {"id": "low-rpm"}, + }, + { + "model_name": "test", + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + router.cache.set_cache(key="low-rpm", value=1, local_only=True) + + filtered = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert [d["model_info"]["id"] for d in filtered] == ["no-rpm"] + + +def test_no_rpm_cache_read_for_usage_based_routing_v2(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1}, + "model_info": {"id": "low-rpm"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + routing_strategy="usage-based-routing-v2", + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments) + + assert get_cache_calls == [], f"Expected zero cache reads under usage-based-routing-v2, got {get_cache_calls}" + + +def test_token_counting_failure_returns_original_deployments(monkeypatch): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "dep-1", "max_input_tokens": 100}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + def _raise_token_counting(*args, **kwargs): + raise ValueError("boom") + + monkeypatch.setattr(router, "_count_pre_call_check_tokens", _raise_token_counting) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + result = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result == deployments + + +def test_model_info_lookup_failure_is_swallowed(monkeypatch): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "dep-1"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + def _raise_model_info(*args, **kwargs): + raise ValueError("boom") + + monkeypatch.setattr(router, "get_router_model_info", _raise_model_info) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + result = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert [d["model_info"]["id"] for d in result] == ["dep-1"] + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 6251142e3868d67c9a8e00430e0ffec0b5a1ddfc Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 14 Oct 2026 12:58:50 +0100 Subject: [PATCH 5/7] fix(router): read rpm through a typed mapping so the skip stays type-clean --- litellm/router.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d7a3e54fac0..c6dd4e2d08b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14022,6 +14022,16 @@ class Router: if param in _PRE_CALL_CHECK_FILTERED_PARAMS and not isinstance(value, str) ) + def _deployment_configures_rpm(self, deployment: object) -> bool: + if not isinstance(deployment, dict): + return False + fields: Final[Mapping[str, object]] = cast(Mapping[str, object], deployment) # cast-ok: untyped dict + params: Final = fields.get("litellm_params") + if not isinstance(params, dict): + return False + parsed_params: Final[Mapping[str, object]] = cast(Mapping[str, object], params) # cast-ok: untyped dict + return parsed_params.get("rpm") is not None + def _pre_call_checks( self, model: str, @@ -14080,18 +14090,19 @@ class Router: # Safe to check statically since `rpm` is a direct litellm_params setting, unlike # max_input_tokens which get_router_model_info() can also derive from the cost map. any_deployment_has_rpm: Final = self.routing_strategy != "usage-based-routing-v2" and any( - isinstance(litellm_params := d.get("litellm_params"), dict) and litellm_params.get("rpm") is not None - for d in _returned_deployments + self._deployment_configures_rpm( + deployment # pyright: ignore[reportUnknownArgumentType] # untyped deployment list + ) + for deployment in _returned_deployments # pyright: ignore[reportUnknownVariableType] # untyped list ) ## get model group RPM ## rpm_key: Final = f"{model}:rpm:{get_utc_datetime().strftime('%H-%M')}" - cached_model_group: Final = ( - self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) + model_group_cache: Final = ( + (self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}) if any_deployment_has_rpm - else None + else {} ) - model_group_cache: Final = cached_model_group if isinstance(cached_model_group, dict) else {} for idx, deployment in enumerate(_returned_deployments): # Cache nested dict access to avoid repeated temporary dict allocations _litellm_params = deployment.get("litellm_params", {}) @@ -14115,7 +14126,7 @@ class Router: continue ## RPM CHECK ## - if any_deployment_has_rpm: + if any_deployment_has_rpm and isinstance(model_group_cache, dict): model_id = _model_info.get("id", "") ### get local router cache ### current_request_cache_local = ( From fb20224595b1334d977b61ec80225ed634bcc420 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Wed, 14 Oct 2026 13:15:48 +0100 Subject: [PATCH 6/7] test(router): record the rpm helper as covered by the cache skip tests --- tests/code_coverage_tests/router_code_coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 7809a23e955..11cffd37b17 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -120,6 +120,7 @@ ignored_function_names = [ "_pinned_deployment", # Tested through the deployment-id scheduler test in test_router.py "_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py + "_deployment_configures_rpm", # Tested through the rpm cache skip tests in test_router.py "_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py From e50b88f74335c6f6dd5c1de006233d82e01cfd27 Mon Sep 17 00:00:00 2001 From: TobiMayr Date: Tue, 27 Oct 2026 12:30:51 +0000 Subject: [PATCH 7/7] test(router): cover the rpm helper's malformed-deployment guards Codecov was failing the patch check on the two early returns that ignore a deployment which cannot carry rpm. Co-authored-by: Cursor --- tests/unit/test_router/test_router.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 40d60214b4c..7d6651b93f1 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -24791,6 +24791,24 @@ def test_no_rpm_cache_read_when_no_deployment_has_rpm(): assert get_cache_calls == [], f"Expected zero cache reads, got {get_cache_calls}" +def test_rpm_helper_ignores_deployments_that_cannot_carry_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "plain"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + assert router._deployment_configures_rpm(["not", "a", "deployment"]) is False + assert router._deployment_configures_rpm({"litellm_params": None}) is False + assert router._deployment_configures_rpm({"litellm_params": {"rpm": 5}}) is True + + def test_rpm_cache_still_read_and_enforced_when_a_deployment_has_rpm(): router = Router( model_list=[