mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): count embedding input as text in pre-call checks
`_count_pre_call_check_tokens` treated every non-null `input` as Responses API input and ran it through `transform_responses_api_input_to_messages`, which calls `.get()` on each item. An embedding batch is a `list[str]`, so that raised `AttributeError: 'str' object has no attribute 'get'`. `_pre_call_checks` catches the exception and returns the unfiltered deployment list, so the request still succeeded but context-window filtering was skipped entirely and an error was logged for every affected request. A `list[str]` now goes to `litellm.token_counter(text=...)`, which already handles both a string and a list of strings. Responses API items and plain strings keep the existing transform path, since the transform accepts both and only breaks on lists of bare strings.
This commit is contained in:
parent
b3882d8e43
commit
d658f0f297
2 changed files with 62 additions and 0 deletions
|
|
@ -12099,6 +12099,10 @@ class Router:
|
|||
Prompt content the message list never carries is read from `request_kwargs`:
|
||||
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
|
||||
Anthropic Messages top-level `system` block.
|
||||
|
||||
Embeddings also arrive as `input`, but as a `list[str]` batch rather than Responses
|
||||
input items. Those are counted as text, since the Responses transform expects
|
||||
object-shaped items and raises on plain strings.
|
||||
"""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
|
||||
anthropic_system_to_openai_message,
|
||||
|
|
@ -12118,6 +12122,10 @@ class Router:
|
|||
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
|
||||
return litellm.token_counter(messages=counted_messages, tools=tools)
|
||||
if input is not None:
|
||||
if isinstance(input, list) and all(isinstance(item, str) for item in input):
|
||||
text_batch = cast(list[str], input) # cast-ok: the isinstance guard narrows every item to str
|
||||
return litellm.token_counter(text=text_batch)
|
||||
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
|
|||
|
|
@ -15679,3 +15679,57 @@ async def test_an_open_circuit_breaker_skips_the_session_binding_without_a_warni
|
|||
assert binding is None
|
||||
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
|
||||
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
def test_count_pre_call_check_tokens_counts_embedding_batches():
|
||||
"""An embedding `input` is a `list[str]`, not Responses API input items. Routing it through the
|
||||
Responses transform raised `AttributeError: 'str' object has no attribute 'get'`, which
|
||||
`_pre_call_checks` swallowed, so context-window filtering was silently skipped and an error was
|
||||
logged per request."""
|
||||
from litellm.router import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{"model_name": "embed-test", "litellm_params": {"model": "vertex_ai/text-embedding-005"}}
|
||||
]
|
||||
)
|
||||
|
||||
batch = router._count_pre_call_check_tokens(messages=None, input=["hello world", "second string"])
|
||||
single = router._count_pre_call_check_tokens(messages=None, input=["hello world"])
|
||||
assert batch > single > 0, "a batch must cost more than one of its own entries"
|
||||
|
||||
assert router._count_pre_call_check_tokens(messages=None, input="single string") > 0
|
||||
assert router._count_pre_call_check_tokens(messages=None, input=[{"role": "user", "content": "hi"}]) > 0
|
||||
assert router._count_pre_call_check_tokens(messages=[{"role": "user", "content": "hi"}], input=None) > 0
|
||||
|
||||
|
||||
def test_pre_call_checks_filters_embedding_over_context_window():
|
||||
"""The consequence the bug hid: with the count raising, every deployment survived the filter.
|
||||
A batch over `max_input_tokens` must now be dropped."""
|
||||
from litellm.router import Router
|
||||
|
||||
over_limit = ["token " * 50, "token " * 50]
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed-test",
|
||||
"litellm_params": {"model": "vertex_ai/text-embedding-005"},
|
||||
"model_info": {"id": "small", "base_model": "vertex_ai/text-embedding-005", "max_input_tokens": 8},
|
||||
}
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
deployments = router.model_list
|
||||
|
||||
with pytest.raises(litellm.exceptions.ContextWindowExceededError):
|
||||
router._pre_call_checks(
|
||||
model="embed-test",
|
||||
healthy_deployments=deployments,
|
||||
messages=None,
|
||||
input=over_limit,
|
||||
)
|
||||
|
||||
kept = router._pre_call_checks(
|
||||
model="embed-test", healthy_deployments=deployments, messages=None, input=["hi"]
|
||||
)
|
||||
assert len(kept) == 1, "a batch inside the limit must survive the filter"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue