mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): count pre-tokenized embedding input in pre-call checks
`AllEmbeddingInputValues` is `str | list[str] | list[int] | list[list[int]]`, but only the `list[str]` batch was routed away from the Responses input transform. The two token-ID shapes still reached `transform_responses_api_input_to_messages`, which calls `.get()` on each item, so they raised `AttributeError: 'int' object has no attribute 'get'` exactly like the string batch did. `_pre_call_checks` swallows it, so context-window filtering was skipped and an error logged per request. Token IDs are already tokens, so the count is the length, not a `token_counter(text=...)` call: that returns 0 for a list of ints. The shape dispatch moves to a module-level helper that returns None when `input` is not an embedding shape, leaving Responses items and plain strings on the transform path.
This commit is contained in:
parent
8832fa1b4d
commit
725d15b021
2 changed files with 31 additions and 4 deletions
|
|
@ -384,6 +384,16 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
|
|||
return False
|
||||
|
||||
|
||||
def _count_embedding_input_tokens(items: Sequence[object]) -> int | None:
|
||||
if all(isinstance(item, str) for item in items):
|
||||
return litellm.token_counter(text=[item for item in items if isinstance(item, str)])
|
||||
if all(isinstance(item, int) for item in items):
|
||||
return len(items)
|
||||
if all(isinstance(item, Sequence) and all(isinstance(token, int) for token in item) for item in items):
|
||||
return sum(len(item) for item in items if isinstance(item, Sequence))
|
||||
return None
|
||||
|
||||
|
||||
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
|
||||
|
||||
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
|
||||
|
|
@ -12100,7 +12110,9 @@ class Router:
|
|||
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
|
||||
Anthropic Messages top-level `system` block.
|
||||
|
||||
Embeddings send `input` as a `list[str]` batch, counted as text.
|
||||
Embeddings send `input` as one of the `AllEmbeddingInputValues` shapes: a `list[str]` batch,
|
||||
counted as text, or pre-tokenized `list[int]` / `list[list[int]]`, whose token count is its
|
||||
own length. None of those are Responses input items, so they never reach the transform.
|
||||
"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
|
||||
anthropic_system_to_openai_message,
|
||||
|
|
@ -12120,9 +12132,10 @@ class Router:
|
|||
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
|
||||
return litellm.token_counter(messages=counted_messages, tools=tools)
|
||||
if input is not None:
|
||||
if isinstance(input, list) and all(isinstance(item, str) for item in input):
|
||||
text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str
|
||||
return litellm.token_counter(text=text_batch)
|
||||
if isinstance(input, list):
|
||||
embedding_tokens: Final = _count_embedding_input_tokens(input)
|
||||
if embedding_tokens is not None:
|
||||
return embedding_tokens
|
||||
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
|
|
|
|||
|
|
@ -15699,6 +15699,20 @@ def test_count_pre_call_check_tokens_counts_embedding_batches():
|
|||
assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0
|
||||
|
||||
|
||||
def test_count_pre_call_check_tokens_counts_pre_tokenized_embedding_input():
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}}
|
||||
]
|
||||
)
|
||||
|
||||
assert router._count_pre_call_check_tokens(messages=None, input=[15496, 2159, 0]) == 3
|
||||
assert router._count_pre_call_check_tokens(messages=None, input=[[15496, 2159], [0, 1, 2]]) == 5
|
||||
assert router._count_pre_call_check_tokens(messages=None, input=[]) == 0
|
||||
|
||||
|
||||
def test_pre_call_checks_filters_embedding_over_context_window():
|
||||
from litellm.router import Router
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue