scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 165

fix: preserve legacy embeddings-shape estimate when call_type is unset

Browse source
This commit is contained in:
Devin AI 2026-07-26 22:41:25 +00:00
parent 81324aef21
commit 1344aa4ee0
1 changed files with 14 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -529,13 +529,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data.get("max_tokens") or data.get("max_completion_tokens") or data.get("max_output_tokens")
)
is_embedding = call_type in (CallTypes.embedding.value, CallTypes.aembedding.value)
match call_type:
case CallTypes.embedding.value | CallTypes.aembedding.value:
has_output_budget = False
case CallTypes.responses.value | CallTypes.aresponses.value:
has_output_budget = True
case _:
# Unknown call type: fall back to the legacy shape heuristic —
# a bare ``input`` (no messages/prompt) is embeddings-style with
# no output budget. Responses traffic always arrives with
# call_type set, so it never lands here.
has_output_budget = not (input_text is not None and not messages and not prompt)
match (explicit_max_tokens, is_embedding):
match (explicit_max_tokens, has_output_budget):
case (mt, _) if mt is not None:
max_tokens_estimate = int(mt)
case (_, True):
# Embeddings have no output tokens
case (_, False):
# No output tokens (embeddings, or an embeddings-shaped request)
max_tokens_estimate = 0
case _ if total_chars == 0:
# Fully contentless request (no messages, prompt, or input).