scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-12 03:48:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 57

fix parallel request limiter use correct user id

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-26 15:03:40 -07:00
parent 16c0307eab
commit 0860599708
2 changed files with 13 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/model_prices_and_context_window_backup.json
View file

@ -2454,6 +2454,17 @@
"mode": "chat",
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models"
},
"vertex_ai/meta/llama-3.2-90b-vision-instruct-maas": {
"max_tokens": 8192,
"max_input_tokens": 128000,
"max_output_tokens": 8192,
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai-llama_models",
"mode": "chat",
"supports_system_messages": true,
"source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas"
},
"vertex_ai/mistral-large@latest": {
"max_tokens": 8191,
"max_input_tokens": 128000,

3
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -322,8 +322,9 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# check if REQUEST ALLOWED for user_id
user_id = user_api_key_dict.user_id
if user_id is not None:
request_count_api_key = f"{user_id}::{precise_minute}::request_count"
_user_id_rate_limits = await self.internal_usage_cache.async_get_cache(
key=user_id,
key=request_count_api_key,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
)
# get user tpm/rpm limits