mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge e50b88f743 into cfe0100e1f
This commit is contained in:
commit
089a2a0fe0
3 changed files with 202 additions and 12 deletions
|
|
@ -14409,6 +14409,16 @@ class Router:
|
|||
if param in _PRE_CALL_CHECK_FILTERED_PARAMS and not isinstance(value, str)
|
||||
)
|
||||
|
||||
def _deployment_configures_rpm(self, deployment: object) -> bool:
|
||||
if not isinstance(deployment, dict):
|
||||
return False
|
||||
fields: Final[Mapping[str, object]] = cast(Mapping[str, object], deployment) # cast-ok: untyped dict
|
||||
params: Final = fields.get("litellm_params")
|
||||
if not isinstance(params, dict):
|
||||
return False
|
||||
parsed_params: Final[Mapping[str, object]] = cast(Mapping[str, object], params) # cast-ok: untyped dict
|
||||
return parsed_params.get("rpm") is not None
|
||||
|
||||
def _pre_call_checks(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -14464,13 +14474,22 @@ class Router:
|
|||
else frozenset()
|
||||
)
|
||||
|
||||
# Safe to check statically since `rpm` is a direct litellm_params setting, unlike
|
||||
# max_input_tokens which get_router_model_info() can also derive from the cost map.
|
||||
any_deployment_has_rpm: Final = self.routing_strategy != "usage-based-routing-v2" and any(
|
||||
self._deployment_configures_rpm(
|
||||
deployment # pyright: ignore[reportUnknownArgumentType] # untyped deployment list
|
||||
)
|
||||
for deployment in _returned_deployments # pyright: ignore[reportUnknownVariableType] # untyped list
|
||||
)
|
||||
|
||||
## get model group RPM ##
|
||||
dt: Final = get_utc_datetime()
|
||||
current_minute: Final = dt.strftime("%H-%M")
|
||||
rpm_key: Final = f"{model}:rpm:{current_minute}"
|
||||
rpm_key: Final = f"{model}:rpm:{get_utc_datetime().strftime('%H-%M')}"
|
||||
model_group_cache: Final = (
|
||||
self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}
|
||||
) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache.
|
||||
(self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {})
|
||||
if any_deployment_has_rpm
|
||||
else {}
|
||||
)
|
||||
for idx, deployment in enumerate(_returned_deployments):
|
||||
# Cache nested dict access to avoid repeated temporary dict allocations
|
||||
_litellm_params = deployment.get("litellm_params", {})
|
||||
|
|
@ -14493,14 +14512,14 @@ class Router:
|
|||
)
|
||||
continue
|
||||
|
||||
model_id = _model_info.get("id", "")
|
||||
## RPM CHECK ##
|
||||
### get local router cache ###
|
||||
current_request_cache_local = (
|
||||
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
|
||||
)
|
||||
### get usage based cache ###
|
||||
if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2":
|
||||
if any_deployment_has_rpm and isinstance(model_group_cache, dict):
|
||||
model_id = _model_info.get("id", "")
|
||||
### get local router cache ###
|
||||
current_request_cache_local = (
|
||||
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
|
||||
)
|
||||
### get usage based cache ###
|
||||
model_group_cache[model_id] = model_group_cache.get(model_id, 0)
|
||||
|
||||
current_request = max(current_request_cache_local, model_group_cache[model_id])
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ ignored_function_names = [
|
|||
"_pinned_deployment", # Tested through the deployment-id scheduler test in test_router.py
|
||||
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
|
||||
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
|
||||
"_deployment_configures_rpm", # Tested through the rpm cache skip tests in test_router.py
|
||||
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
|
||||
|
|
|
|||
|
|
@ -24997,6 +24997,176 @@ class TestPreCallChecksOptimization:
|
|||
assert deployments[0].get("model_info", {}).get("id") == "small", "First deployment ID changed!"
|
||||
assert deployments[1].get("model_info", {}).get("id") == "large", "Second deployment ID changed!"
|
||||
|
||||
|
||||
def _rpm_cache_keys_read_by(router: Router, model: str, deployments: list) -> list:
|
||||
get_cache_calls: list = []
|
||||
original_get_cache = router.cache.get_cache
|
||||
|
||||
def _tracking_get_cache(*args, **kwargs):
|
||||
get_cache_calls.append(kwargs.get("key", args[0] if args else None))
|
||||
return original_get_cache(*args, **kwargs)
|
||||
|
||||
router.cache.get_cache = _tracking_get_cache
|
||||
router._pre_call_checks(
|
||||
model=model,
|
||||
healthy_deployments=deployments,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
return get_cache_calls
|
||||
|
||||
|
||||
def test_no_rpm_cache_read_when_no_deployment_has_rpm():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
|
||||
"model_info": {"id": "no-rpm-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"},
|
||||
"model_info": {"id": "no-rpm-2"},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
deployments = router.get_model_list(model_name="test")
|
||||
assert deployments is not None
|
||||
|
||||
get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments)
|
||||
|
||||
assert get_cache_calls == [], f"Expected zero cache reads, got {get_cache_calls}"
|
||||
|
||||
|
||||
def test_rpm_helper_ignores_deployments_that_cannot_carry_rpm():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
|
||||
"model_info": {"id": "plain"},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
assert router._deployment_configures_rpm(["not", "a", "deployment"]) is False
|
||||
assert router._deployment_configures_rpm({"litellm_params": None}) is False
|
||||
assert router._deployment_configures_rpm({"litellm_params": {"rpm": 5}}) is True
|
||||
|
||||
|
||||
def test_rpm_cache_still_read_and_enforced_when_a_deployment_has_rpm():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1},
|
||||
"model_info": {"id": "low-rpm"},
|
||||
},
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"},
|
||||
"model_info": {"id": "no-rpm"},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
deployments = router.get_model_list(model_name="test")
|
||||
assert deployments is not None
|
||||
router.cache.set_cache(key="low-rpm", value=1, local_only=True)
|
||||
|
||||
filtered = router._pre_call_checks(
|
||||
model="test",
|
||||
healthy_deployments=deployments,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert [d["model_info"]["id"] for d in filtered] == ["no-rpm"]
|
||||
|
||||
|
||||
def test_no_rpm_cache_read_for_usage_based_routing_v2():
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test", "rpm": 1},
|
||||
"model_info": {"id": "low-rpm"},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
routing_strategy="usage-based-routing-v2",
|
||||
)
|
||||
deployments = router.get_model_list(model_name="test")
|
||||
assert deployments is not None
|
||||
|
||||
get_cache_calls = _rpm_cache_keys_read_by(router, "test", deployments)
|
||||
|
||||
assert get_cache_calls == [], f"Expected zero cache reads under usage-based-routing-v2, got {get_cache_calls}"
|
||||
|
||||
|
||||
def test_token_counting_failure_returns_original_deployments(monkeypatch):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
|
||||
"model_info": {"id": "dep-1", "max_input_tokens": 100},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
def _raise_token_counting(*args, **kwargs):
|
||||
raise ValueError("boom")
|
||||
|
||||
monkeypatch.setattr(router, "_count_pre_call_check_tokens", _raise_token_counting)
|
||||
deployments = router.get_model_list(model_name="test")
|
||||
assert deployments is not None
|
||||
|
||||
result = router._pre_call_checks(
|
||||
model="test",
|
||||
healthy_deployments=deployments,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert result == deployments
|
||||
|
||||
|
||||
def test_model_info_lookup_failure_is_swallowed(monkeypatch):
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test",
|
||||
"litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"},
|
||||
"model_info": {"id": "dep-1"},
|
||||
},
|
||||
],
|
||||
set_verbose=False,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
def _raise_model_info(*args, **kwargs):
|
||||
raise ValueError("boom")
|
||||
|
||||
monkeypatch.setattr(router, "get_router_model_info", _raise_model_info)
|
||||
deployments = router.get_model_list(model_name="test")
|
||||
assert deployments is not None
|
||||
|
||||
result = router._pre_call_checks(
|
||||
model="test",
|
||||
healthy_deployments=deployments,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert [d["model_info"]["id"] for d in result] == ["dep-1"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue