fix(router): count pre-tokenized embedding input in pre-call checks

`AllEmbeddingInputValues` is `str | list[str] | list[int] | list[list[int]]`, but only
the `list[str]` batch was routed away from the Responses input transform. The two
token-ID shapes still reached `transform_responses_api_input_to_messages`, which calls
`.get()` on each item, so they raised `AttributeError: 'int' object has no attribute
'get'` exactly like the string batch did. `_pre_call_checks` swallows it, so
context-window filtering was skipped and an error logged per request.

Token IDs are already tokens, so the count is the length, not a `token_counter(text=...)`
call: that returns 0 for a list of ints. The shape dispatch moves to a module-level
helper that returns None when `input` is not an embedding shape, leaving Responses items
and plain strings on the transform path.
This commit is contained in:
Isaac Hernandez 2026-08-10 12:11:11 -04:00
parent 8832fa1b4d
commit 725d15b021
2 changed files with 31 additions and 4 deletions

View file

@ -384,6 +384,16 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None
return False
def _count_embedding_input_tokens(items: Sequence[object]) -> int | None:
if all(isinstance(item, str) for item in items):
return litellm.token_counter(text=[item for item in items if isinstance(item, str)])
if all(isinstance(item, int) for item in items):
return len(items)
if all(isinstance(item, Sequence) and all(isinstance(token, int) for token in item) for item in items):
return sum(len(item) for item in items if isinstance(item, Sequence))
return None
_PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT")
_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"})
@ -12100,7 +12110,9 @@ class Router:
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
Anthropic Messages top-level `system` block.
Embeddings send `input` as a `list[str]` batch, counted as text.
Embeddings send `input` as one of the `AllEmbeddingInputValues` shapes: a `list[str]` batch,
counted as text, or pre-tokenized `list[int]` / `list[list[int]]`, whose token count is its
own length. None of those are Responses input items, so they never reach the transform.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
anthropic_system_to_openai_message,
@ -12120,9 +12132,10 @@ class Router:
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
return litellm.token_counter(messages=counted_messages, tools=tools)
if input is not None:
if isinstance(input, list) and all(isinstance(item, str) for item in input):
text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str
return litellm.token_counter(text=text_batch)
if isinstance(input, list):
embedding_tokens: Final = _count_embedding_input_tokens(input)
if embedding_tokens is not None:
return embedding_tokens
from openai.types.responses.response_create_params import ResponseInputParam

View file

@ -15699,6 +15699,20 @@ def test_count_pre_call_check_tokens_counts_embedding_batches():
assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0
def test_count_pre_call_check_tokens_counts_pre_tokenized_embedding_input():
from litellm.router import Router
router = Router(
model_list=[
{"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}}
]
)
assert router._count_pre_call_check_tokens(messages=None, input=[15496, 2159, 0]) == 3
assert router._count_pre_call_check_tokens(messages=None, input=[[15496, 2159], [0, 1, 2]]) == 5
assert router._count_pre_call_check_tokens(messages=None, input=[]) == 0
def test_pre_call_checks_filters_embedding_over_context_window():
from litellm.router import Router