diff --git a/litellm/router.py b/litellm/router.py index 0d75fbbad18..352603713e4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -384,6 +384,16 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None return False +def _count_embedding_input_tokens(items: Sequence[object]) -> int | None: + if all(isinstance(item, str) for item in items): + return litellm.token_counter(text=[item for item in items if isinstance(item, str)]) + if all(isinstance(item, int) for item in items): + return len(items) + if all(isinstance(item, Sequence) and all(isinstance(token, int) for token in item) for item in items): + return sum(len(item) for item in items if isinstance(item, Sequence)) + return None + + _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) @@ -12100,7 +12110,9 @@ class Router: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. - Embeddings send `input` as a `list[str]` batch, counted as text. + Embeddings send `input` as one of the `AllEmbeddingInputValues` shapes: a `list[str]` batch, + counted as text, or pre-tokenized `list[int]` / `list[list[int]]`, whose token count is its + own length. None of those are Responses input items, so they never reach the transform. """ from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( anthropic_system_to_openai_message, @@ -12120,9 +12132,10 @@ class Router: counted_messages: Final = (system_message, *messages) if system_message is not None else messages return litellm.token_counter(messages=counted_messages, tools=tools) if input is not None: - if isinstance(input, list) and all(isinstance(item, str) for item in input): - text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str - return litellm.token_counter(text=text_batch) + if isinstance(input, list): + embedding_tokens: Final = _count_embedding_input_tokens(input) + if embedding_tokens is not None: + return embedding_tokens from openai.types.responses.response_create_params import ResponseInputParam diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9bd5bfc75fb..a6dcc4c42dd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15699,6 +15699,20 @@ def test_count_pre_call_check_tokens_counts_embedding_batches(): assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0 +def test_count_pre_call_check_tokens_counts_pre_tokenized_embedding_input(): + from litellm.router import Router + + router = Router( + model_list=[ + {"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}} + ] + ) + + assert router._count_pre_call_check_tokens(messages=None, input=[15496, 2159, 0]) == 3 + assert router._count_pre_call_check_tokens(messages=None, input=[[15496, 2159], [0, 1, 2]]) == 5 + assert router._count_pre_call_check_tokens(messages=None, input=[]) == 0 + + def test_pre_call_checks_filters_embedding_over_context_window(): from litellm.router import Router