scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

perf: eliminate redundant router lookups in v1 parallel request limiter

Browse source 
Compute get_key_model_tpm/rpm_limit once before the guard condition
instead of calling each function twice (once to check non-None, once to
retrieve). Removes 2 extra llm_router.get_model_list() calls per request
when deployment defaults are active.

Co-Authored-By: Claude (claude-sonnet-4-6) <noreply@anthropic.com>
This commit is contained in:
Ephrim Stanley 2026-03-19 01:57:09 -04:00
parent 36dc893770
commit b90f520748
1 changed files with 7 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -296,19 +296,16 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Check if request under RPM/TPM per model for a given API Key
_model = data.get("model", None)
if (
get_key_model_tpm_limit(user_api_key_dict, model_name=_model) is not None
or get_key_model_rpm_limit(user_api_key_dict, model_name=_model) is not None
):
_tpm_limit_for_key_model = get_key_model_tpm_limit(
user_api_key_dict, model_name=_model
)
_rpm_limit_for_key_model = get_key_model_rpm_limit(
user_api_key_dict, model_name=_model
)
if _tpm_limit_for_key_model is not None or _rpm_limit_for_key_model is not None:
request_count_api_key = (
f"{api_key}::{_model}::{precise_minute}::request_count"
)
_tpm_limit_for_key_model = get_key_model_tpm_limit(
user_api_key_dict, model_name=_model
)
_rpm_limit_for_key_model = get_key_model_rpm_limit(
user_api_key_dict, model_name=_model
)
tpm_limit_for_model = None
rpm_limit_for_model = None