From 725d15b021f12a7288e4e38d72eaba4998cef70f Mon Sep 17 00:00:00 2001 From: Isaac Hernandez Date: Mon, 10 Aug 2026 12:11:11 -0400 Subject: [PATCH] fix(router): count pre-tokenized embedding input in pre-call checks `AllEmbeddingInputValues` is `str | list[str] | list[int] | list[list[int]]`, but only the `list[str]` batch was routed away from the Responses input transform. The two token-ID shapes still reached `transform_responses_api_input_to_messages`, which calls `.get()` on each item, so they raised `AttributeError: 'int' object has no attribute 'get'` exactly like the string batch did. `_pre_call_checks` swallows it, so context-window filtering was skipped and an error logged per request. Token IDs are already tokens, so the count is the length, not a `token_counter(text=...)` call: that returns 0 for a list of ints. The shape dispatch moves to a module-level helper that returns None when `input` is not an embedding shape, leaving Responses items and plain strings on the transform path. --- litellm/router.py | 21 +++++++++++++++++---- tests/test_litellm/test_router.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0d75fbbad18..352603713e4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -384,6 +384,16 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None return False +def _count_embedding_input_tokens(items: Sequence[object]) -> int | None: + if all(isinstance(item, str) for item in items): + return litellm.token_counter(text=[item for item in items if isinstance(item, str)]) + if all(isinstance(item, int) for item in items): + return len(items) + if all(isinstance(item, Sequence) and all(isinstance(token, int) for token in item) for item in items): + return sum(len(item) for item in items if isinstance(item, Sequence)) + return None + + _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) @@ -12100,7 +12110,9 @@ class Router: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. - Embeddings send `input` as a `list[str]` batch, counted as text. + Embeddings send `input` as one of the `AllEmbeddingInputValues` shapes: a `list[str]` batch, + counted as text, or pre-tokenized `list[int]` / `list[list[int]]`, whose token count is its + own length. None of those are Responses input items, so they never reach the transform. """ from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( anthropic_system_to_openai_message, @@ -12120,9 +12132,10 @@ class Router: counted_messages: Final = (system_message, *messages) if system_message is not None else messages return litellm.token_counter(messages=counted_messages, tools=tools) if input is not None: - if isinstance(input, list) and all(isinstance(item, str) for item in input): - text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str - return litellm.token_counter(text=text_batch) + if isinstance(input, list): + embedding_tokens: Final = _count_embedding_input_tokens(input) + if embedding_tokens is not None: + return embedding_tokens from openai.types.responses.response_create_params import ResponseInputParam diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9bd5bfc75fb..a6dcc4c42dd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15699,6 +15699,20 @@ def test_count_pre_call_check_tokens_counts_embedding_batches(): assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0 +def test_count_pre_call_check_tokens_counts_pre_tokenized_embedding_input(): + from litellm.router import Router + + router = Router( + model_list=[ + {"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}} + ] + ) + + assert router._count_pre_call_check_tokens(messages=None, input=[15496, 2159, 0]) == 3 + assert router._count_pre_call_check_tokens(messages=None, input=[[15496, 2159], [0, 1, 2]]) == 5 + assert router._count_pre_call_check_tokens(messages=None, input=[]) == 0 + + def test_pre_call_checks_filters_embedding_over_context_window(): from litellm.router import Router