This commit is contained in:
Tobias Mayr 2026-10-27 12:31:01 +00:00 • committed by GitHub
commit 089a2a0fe0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 202 additions and 12 deletions

View file

@ -14409,6 +14409,16 @@ class Router:
if param in _PRE_CALL_CHECK_FILTERED_PARAMS and not isinstance(value, str)
)
def _deployment_configures_rpm(self, deployment: object) -> bool:
if not isinstance(deployment, dict):
return False
fields: Final[Mapping[str, object]] = cast(Mapping[str, object], deployment) # cast-ok: untyped dict
params: Final = fields.get("litellm_params")
if not isinstance(params, dict):
return False
parsed_params: Final[Mapping[str, object]] = cast(Mapping[str, object], params) # cast-ok: untyped dict
return parsed_params.get("rpm") is not None
def _pre_call_checks(
self,
model: str,
@ -14464,13 +14474,22 @@ class Router:
else frozenset()
)
# Safe to check statically since `rpm` is a direct litellm_params setting, unlike
# max_input_tokens which get_router_model_info() can also derive from the cost map.
any_deployment_has_rpm: Final = self.routing_strategy != "usage-based-routing-v2" and any(
self._deployment_configures_rpm(
deployment # pyright: ignore[reportUnknownArgumentType] # untyped deployment list
)
for deployment in _returned_deployments # pyright: ignore[reportUnknownVariableType] # untyped list
)
## get model group RPM ##
dt: Final = get_utc_datetime()
current_minute: Final = dt.strftime("%H-%M")
rpm_key: Final = f"{model}:rpm:{current_minute}"
rpm_key: Final = f"{model}:rpm:{get_utc_datetime().strftime('%H-%M')}"
model_group_cache: Final = (
self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}
) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache.
(self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {})
if any_deployment_has_rpm
else {}
)
for idx, deployment in enumerate(_returned_deployments):
# Cache nested dict access to avoid repeated temporary dict allocations
_litellm_params = deployment.get("litellm_params", {})
@ -14493,14 +14512,14 @@ class Router:
)
continue
model_id = _model_info.get("id", "")
## RPM CHECK ##
### get local router cache ###
current_request_cache_local = (
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
)
### get usage based cache ###
if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2":
if any_deployment_has_rpm and isinstance(model_group_cache, dict):
model_id = _model_info.get("id", "")
### get local router cache ###
current_request_cache_local = (
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
)
### get usage based cache ###
model_group_cache[model_id] = model_group_cache.get(model_id, 0)
current_request = max(current_request_cache_local, model_group_cache[model_id])

View file

@ -121,6 +121,7 @@ ignored_function_names = [
"_pinned_deployment", # Tested through the deployment-id scheduler test in test_router.py
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
"_deployment_configures_rpm", # Tested through the rpm cache skip tests in test_router.py
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py

View file

@ -24997,6 +24997,176 @@ class TestPreCallChecksOptimization:
assert deployments[0].get("model_info", {}).get("id") == "small", "First deployment ID changed!"
assert deployments[1].get("model_info", {}).get("id") == "large", "Second deployment ID changed!"
def _rpm_cache_keys_read_by(router: Router, model: str, deployments: list) -> list:
get_cache_calls: list = []
original_get_cache = router.cache.get_cache
def _tracking_get_cache(*args, **kwargs):
get_cache_calls.append(kwargs.get("key", args[0] if args else None))
return original_get_cache(*args, **kwargs)
router.cache.get_cache = _tracking_get_cache
router._pre_call_checks(
model=model,
healthy_deployments=deployments,
messages=[{"role": "user", "content": "hi"}],
)
return get_cache_calls
def test_no_rpm_cache_read_when_no_deployment_has_rpm():
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
"model_info": {"id": "no-rpm-1"},
},
{
"model_name": "test",
"litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"},
"model_info": {"id": "no-rpm-2"},
},
],
set_verbose=False,
enable_pre_call_checks=True,
)
deployments = router.get_model_list(model_name="test")
assert deployments is not None
get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments)
assert get_cache_calls == [], f"Expected zero cache reads, got {get_cache_calls}"
def test_rpm_helper_ignores_deployments_that_cannot_carry_rpm():
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
"model_info": {"id": "plain"},
},
],
set_verbose=False,
enable_pre_call_checks=True,
)
assert router._deployment_configures_rpm(["not", "a", "deployment"]) is False
assert router._deployment_configures_rpm({"litellm_params": None}) is False
assert router._deployment_configures_rpm({"litellm_params": {"rpm": 5}}) is True
def test_rpm_cache_still_read_and_enforced_when_a_deployment_has_rpm():
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1},
"model_info": {"id": "low-rpm"},
},
{
"model_name": "test",
"litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"},
"model_info": {"id": "no-rpm"},
},
],
set_verbose=False,
enable_pre_call_checks=True,
)
deployments = router.get_model_list(model_name="test")
assert deployments is not None
router.cache.set_cache(key="low-rpm", value=1, local_only=True)
filtered = router._pre_call_checks(
model="test",
healthy_deployments=deployments,
messages=[{"role": "user", "content": "hi"}],
)
assert [d["model_info"]["id"] for d in filtered] == ["no-rpm"]
def test_no_rpm_cache_read_for_usage_based_routing_v2():
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1},
"model_info": {"id": "low-rpm"},
},
],
set_verbose=False,
enable_pre_call_checks=True,
routing_strategy="usage-based-routing-v2",
)
deployments = router.get_model_list(model_name="test")
assert deployments is not None
get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments)
assert get_cache_calls == [], f"Expected zero cache reads under usage-based-routing-v2, got {get_cache_calls}"
def test_token_counting_failure_returns_original_deployments(monkeypatch):
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
"model_info": {"id": "dep-1", "max_input_tokens": 100},
},
],
set_verbose=False,
enable_pre_call_checks=True,
)
def _raise_token_counting(*args, **kwargs):
raise ValueError("boom")
monkeypatch.setattr(router, "_count_pre_call_check_tokens", _raise_token_counting)
deployments = router.get_model_list(model_name="test")
assert deployments is not None
result = router._pre_call_checks(
model="test",
healthy_deployments=deployments,
messages=[{"role": "user", "content": "hi"}],
)
assert result == deployments
def test_model_info_lookup_failure_is_swallowed(monkeypatch):
router = Router(
model_list=[
{
"model_name": "test",
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
"model_info": {"id": "dep-1"},
},
],
set_verbose=False,
enable_pre_call_checks=True,
)
def _raise_model_info(*args, **kwargs):
raise ValueError("boom")
monkeypatch.setattr(router, "get_router_model_info", _raise_model_info)
deployments = router.get_model_list(model_name="test")
assert deployments is not None
result = router._pre_call_checks(
model="test",
healthy_deployments=deployments,
messages=[{"role": "user", "content": "hi"}],
)
assert [d["model_info"]["id"] for d in result] == ["dep-1"]
if __name__ == "__main__":
pytest.main([__file__, "-v"])