diff --git a/litellm/router.py b/litellm/router.py index abcdf9068d9..d964785c2aa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14409,6 +14409,16 @@ class Router: if param in _PRE_CALL_CHECK_FILTERED_PARAMS and not isinstance(value, str) ) + def _deployment_configures_rpm(self, deployment: object) -> bool: + if not isinstance(deployment, dict): + return False + fields: Final[Mapping[str, object]] = cast(Mapping[str, object], deployment) # cast-ok: untyped dict + params: Final = fields.get("litellm_params") + if not isinstance(params, dict): + return False + parsed_params: Final[Mapping[str, object]] = cast(Mapping[str, object], params) # cast-ok: untyped dict + return parsed_params.get("rpm") is not None + def _pre_call_checks( self, model: str, @@ -14464,13 +14474,22 @@ class Router: else frozenset() ) + # Safe to check statically since `rpm` is a direct litellm_params setting, unlike + # max_input_tokens which get_router_model_info() can also derive from the cost map. + any_deployment_has_rpm: Final = self.routing_strategy != "usage-based-routing-v2" and any( + self._deployment_configures_rpm( + deployment # pyright: ignore[reportUnknownArgumentType] # untyped deployment list + ) + for deployment in _returned_deployments # pyright: ignore[reportUnknownVariableType] # untyped list + ) + ## get model group RPM ## - dt: Final = get_utc_datetime() - current_minute: Final = dt.strftime("%H-%M") - rpm_key: Final = f"{model}:rpm:{current_minute}" + rpm_key: Final = f"{model}:rpm:{get_utc_datetime().strftime('%H-%M')}" model_group_cache: Final = ( - self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {} - ) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache. + (self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}) + if any_deployment_has_rpm + else {} + ) for idx, deployment in enumerate(_returned_deployments): # Cache nested dict access to avoid repeated temporary dict allocations _litellm_params = deployment.get("litellm_params", {}) @@ -14493,14 +14512,14 @@ class Router: ) continue - model_id = _model_info.get("id", "") ## RPM CHECK ## - ### get local router cache ### - current_request_cache_local = ( - self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0 - ) - ### get usage based cache ### - if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2": + if any_deployment_has_rpm and isinstance(model_group_cache, dict): + model_id = _model_info.get("id", "") + ### get local router cache ### + current_request_cache_local = ( + self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0 + ) + ### get usage based cache ### model_group_cache[model_id] = model_group_cache.get(model_id, 0) current_request = max(current_request_cache_local, model_group_cache[model_id]) diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 853b10ad97b..5d40afc9b67 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -121,6 +121,7 @@ ignored_function_names = [ "_pinned_deployment", # Tested through the deployment-id scheduler test in test_router.py "_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py + "_deployment_configures_rpm", # Tested through the rpm cache skip tests in test_router.py "_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..13957eb29ae 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -24997,6 +24997,176 @@ class TestPreCallChecksOptimization: assert deployments[0].get("model_info", {}).get("id") == "small", "First deployment ID changed!" assert deployments[1].get("model_info", {}).get("id") == "large", "Second deployment ID changed!" + +def _rpm_cache_keys_read_by(router: Router, model: str, deployments: list) -> list: + get_cache_calls: list = [] + original_get_cache = router.cache.get_cache + + def _tracking_get_cache(*args, **kwargs): + get_cache_calls.append(kwargs.get("key", args[0] if args else None)) + return original_get_cache(*args, **kwargs) + + router.cache.get_cache = _tracking_get_cache + router._pre_call_checks( + model=model, + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + return get_cache_calls + + +def test_no_rpm_cache_read_when_no_deployment_has_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm-1"}, + }, + { + "model_name": "test", + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm-2"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments) + + assert get_cache_calls == [], f"Expected zero cache reads, got {get_cache_calls}" + + +def test_rpm_helper_ignores_deployments_that_cannot_carry_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "plain"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + assert router._deployment_configures_rpm(["not", "a", "deployment"]) is False + assert router._deployment_configures_rpm({"litellm_params": None}) is False + assert router._deployment_configures_rpm({"litellm_params": {"rpm": 5}}) is True + + +def test_rpm_cache_still_read_and_enforced_when_a_deployment_has_rpm(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1}, + "model_info": {"id": "low-rpm"}, + }, + { + "model_name": "test", + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"}, + "model_info": {"id": "no-rpm"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + router.cache.set_cache(key="low-rpm", value=1, local_only=True) + + filtered = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert [d["model_info"]["id"] for d in filtered] == ["no-rpm"] + + +def test_no_rpm_cache_read_for_usage_based_routing_v2(): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1}, + "model_info": {"id": "low-rpm"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + routing_strategy="usage-based-routing-v2", + ) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments) + + assert get_cache_calls == [], f"Expected zero cache reads under usage-based-routing-v2, got {get_cache_calls}" + + +def test_token_counting_failure_returns_original_deployments(monkeypatch): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "dep-1", "max_input_tokens": 100}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + def _raise_token_counting(*args, **kwargs): + raise ValueError("boom") + + monkeypatch.setattr(router, "_count_pre_call_check_tokens", _raise_token_counting) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + result = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result == deployments + + +def test_model_info_lookup_failure_is_swallowed(monkeypatch): + router = Router( + model_list=[ + { + "model_name": "test", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, + "model_info": {"id": "dep-1"}, + }, + ], + set_verbose=False, + enable_pre_call_checks=True, + ) + + def _raise_model_info(*args, **kwargs): + raise ValueError("boom") + + monkeypatch.setattr(router, "get_router_model_info", _raise_model_info) + deployments = router.get_model_list(model_name="test") + assert deployments is not None + + result = router._pre_call_checks( + model="test", + healthy_deployments=deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert [d["model_info"]["id"] for d in result] == ["dep-1"] + + if __name__ == "__main__": pytest.main([__file__, "-v"])