scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

refactor: use get_str_from_messages utility for token estimation

Browse source
This commit is contained in:
Harshit Jain 2026-01-12 19:09:46 +05:30
parent 0be5712961
commit 433141c1d3
No known key found for this signature in database
GPG key ID: 36C392CD4415B4CF
1 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -28,6 +28,9 @@ from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
from litellm.types.utils import ModelResponse, Usage
@ -284,14 +287,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
input_text = data.get("input") # For embeddings
if messages:
# Chat completions
# Chat completions - use shared utility for text extraction
try:
total_chars = sum(
len(str(m.get("content", "")))
for m in messages
if isinstance(m, dict)
)
estimated_input_tokens = total_chars // DEFAULT_CHARS_PER_TOKEN
total_text = get_str_from_messages(messages)
total_chars = len(total_text)
estimated_input_tokens = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN)
except Exception as e:
verbose_proxy_logger.debug(
f"Token counting failed, using fallback estimation: {str(e)}"