fix(router): count embedding input as text in pre-call checks

`_count_pre_call_check_tokens` treated every non-null `input` as Responses API
input and ran it through `transform_responses_api_input_to_messages`, which calls
`.get()` on each item. An embedding batch is a `list[str]`, so that raised
`AttributeError: 'str' object has no attribute 'get'`.

`_pre_call_checks` catches the exception and returns the unfiltered deployment
list, so the request still succeeded but context-window filtering was skipped
entirely and an error was logged for every affected request.

A `list[str]` now goes to `litellm.token_counter(text=...)`, which already handles
both a string and a list of strings. Responses API items and plain strings keep
the existing transform path, since the transform accepts both and only breaks on
lists of bare strings.
This commit is contained in:
Isaac Hernandez 2026-08-03 10:17:05 -04:00 • committed by Isaac Hernandez
parent b3882d8e43
commit d658f0f297
2 changed files with 62 additions and 0 deletions

View file

@ -12099,6 +12099,10 @@ class Router:
Prompt content the message list never carries is read from `request_kwargs`:
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
Anthropic Messages top-level `system` block.
Embeddings also arrive as `input`, but as a `list[str]` batch rather than Responses
input items. Those are counted as text, since the Responses transform expects
object-shaped items and raises on plain strings.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
anthropic_system_to_openai_message,
@ -12118,6 +12122,10 @@ class Router:
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
return litellm.token_counter(messages=counted_messages, tools=tools)
if input is not None:
if isinstance(input, list) and all(isinstance(item, str) for item in input):
text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str
return litellm.token_counter(text=text_batch)
from openai.types.responses.response_create_params import ResponseInputParam
from litellm.responses.litellm_completion_transformation.transformation import (

View file

@ -15679,3 +15679,57 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni
assert binding is None
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
def test_count_pre_call_check_tokens_counts_embedding_batches():
"""An embedding `input` is a `list[str]`, not Responses API input items. Routing it through the
Responses transform raised `AttributeError: 'str' object has no attribute 'get'`, which
`_pre_call_checks` swallowed, so context-window filtering was silently skipped and an error was
logged per request."""
from litellm.router import Router
router = Router(
model_list=[
{"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}}
]
)
batch = router._count_pre_call_check_tokens(messages=None, input=["hello world", "second string"])
single = router._count_pre_call_check_tokens(messages=None, input=["hello world"])
assert batch > single > 0, "a batch must cost more than one of its own entries"
assert router._count_pre_call_check_tokens(messages=None, input="single string") > 0
assert router._count_pre_call_check_tokens(messages=None, input=[{"role": "user", "content": "hi"}]) > 0
assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0
def test_pre_call_checks_filters_embedding_over_context_window():
"""The consequence the bug hid: with the count raising, every deployment survived the filter.
A batch over `max_input_tokens` must now be dropped."""
from litellm.router import Router
over_limit = ["token " * 50, "token " * 50]
router = Router(
model_list=[
{
"model_name": "embed-test",
"litellm_params": {"model": "vertex_ai/text-embedding-005"},
"model_info": {"id": "small", "base_model": "vertex_ai/text-embedding-005", "max_input_tokens": 8},
}
],
enable_pre_call_checks=True,
)
deployments = router.model_list
with pytest.raises(litellm.exceptions.ContextWindowExceededError):
router._pre_call_checks(
model="embed-test",
healthy_deployments=deployments,
messages=None,
input=over_limit,
)
kept = router._pre_call_checks(
model="embed-test", healthy_deployments=deployments, messages=None, input=["hi"]
)
assert len(kept) == 1, "a batch inside the limit must survive the filter"