From d658f0f297fc47360698a90b533f03a6873307af Mon Sep 17 00:00:00 2001 From: Isaac Hernandez Date: Mon, 3 Aug 2026 10:17:05 -0400 Subject: [PATCH] fix(router): count embedding input as text in pre-call checks `_count_pre_call_check_tokens` treated every non-null `input` as Responses API input and ran it through `transform_responses_api_input_to_messages`, which calls `.get()` on each item. An embedding batch is a `list[str]`, so that raised `AttributeError: 'str' object has no attribute 'get'`. `_pre_call_checks` catches the exception and returns the unfiltered deployment list, so the request still succeeded but context-window filtering was skipped entirely and an error was logged for every affected request. A `list[str]` now goes to `litellm.token_counter(text=...)`, which already handles both a string and a list of strings. Responses API items and plain strings keep the existing transform path, since the transform accepts both and only breaks on lists of bare strings. --- litellm/router.py | 8 +++++ tests/test_litellm/test_router.py | 54 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..5fbf7c8faa5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12099,6 +12099,10 @@ class Router: Prompt content the message list never carries is read from `request_kwargs`: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. + + Embeddings also arrive as `input`, but as a `list[str]` batch rather than Responses + input items. Those are counted as text, since the Responses transform expects + object-shaped items and raises on plain strings. """ from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( anthropic_system_to_openai_message, @@ -12118,6 +12122,10 @@ class Router: counted_messages: Final = (system_message, *messages) if system_message is not None else messages return litellm.token_counter(messages=counted_messages, tools=tools) if input is not None: + if isinstance(input, list) and all(isinstance(item, str) for item in input): + text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str + return litellm.token_counter(text=text_batch) + from openai.types.responses.response_create_params import ResponseInputParam from litellm.responses.litellm_completion_transformation.transformation import ( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b8a0d70f5bc..3af6353a397 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15679,3 +15679,57 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni assert binding is None assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] assert any("circuit breaker is open" in record.getMessage() for record in caplog.records) + + +def test_count_pre_call_check_tokens_counts_embedding_batches(): + """An embedding `input` is a `list[str]`, not Responses API input items. Routing it through the + Responses transform raised `AttributeError: 'str' object has no attribute 'get'`, which + `_pre_call_checks` swallowed, so context-window filtering was silently skipped and an error was + logged per request.""" + from litellm.router import Router + + router = Router( + model_list=[ + {"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}} + ] + ) + + batch = router._count_pre_call_check_tokens(messages=None, input=["hello world", "second string"]) + single = router._count_pre_call_check_tokens(messages=None, input=["hello world"]) + assert batch > single > 0, "a batch must cost more than one of its own entries" + + assert router._count_pre_call_check_tokens(messages=None, input="single string") > 0 + assert router._count_pre_call_check_tokens(messages=None, input=[{"role": "user", "content": "hi"}]) > 0 + assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0 + + +def test_pre_call_checks_filters_embedding_over_context_window(): + """The consequence the bug hid: with the count raising, every deployment survived the filter. + A batch over `max_input_tokens` must now be dropped.""" + from litellm.router import Router + + over_limit = ["token " * 50, "token " * 50] + router = Router( + model_list=[ + { + "model_name": "embed-test", + "litellm_params": {"model": "vertex_ai/text-embedding-005"}, + "model_info": {"id": "small", "base_model": "vertex_ai/text-embedding-005", "max_input_tokens": 8}, + } + ], + enable_pre_call_checks=True, + ) + deployments = router.model_list + + with pytest.raises(litellm.exceptions.ContextWindowExceededError): + router._pre_call_checks( + model="embed-test", + healthy_deployments=deployments, + messages=None, + input=over_limit, + ) + + kept = router._pre_call_checks( + model="embed-test", healthy_deployments=deployments, messages=None, input=["hi"] + ) + assert len(kept) == 1, "a batch inside the limit must survive the filter"