diff --git a/litellm/router.py b/litellm/router.py index 25591a435f9..1fa03713356 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -46,7 +46,7 @@ import anyio import httpx import openai from openai import AsyncOpenAI -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, StrictInt, StrictStr, TypeAdapter, ValidationError from typing_extensions import assert_never, overload import litellm @@ -604,12 +604,29 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) return False +def _longest_embedding_input_tokens(items: Sequence[object]) -> int | None: + """Token count of the longest input in an embedding batch, or None when `items` is not one. + + An embedding model's context window bounds each input on its own, so the batch is never summed. + """ + if all(isinstance(item, int) for item in items): + return len(items) + try: + batch: Final = _EMBEDDING_BATCH_ADAPTER.validate_python(items) + except ValidationError: + return None + return max((litellm.token_counter(text=item) if isinstance(item, str) else len(item) for item in batch), default=0) + + _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) +_EMBEDDING_BATCH_ADAPTER: Final[TypeAdapter[tuple[StrictStr | tuple[StrictInt, ...], ...]]] = TypeAdapter( + tuple[StrictStr | tuple[StrictInt, ...], ...] +) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -14058,12 +14075,14 @@ class Router: Prompt content the message list never carries is read from `request_kwargs`: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. + + Embedding batches (`list[str]`, `list[int]`, `list[list[int]]`) count their longest input. """ from litellm.llms.anthropic.pass_through.messages.utils import ( anthropic_system_to_openai_message, ) - extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({}) + extras: Final = request_kwargs if request_kwargs is not None else _NO_DEFAULT_PARAMS raw_instructions: Final = extras.get("instructions") instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None raw_tools: Final = extras.get("tools") @@ -14076,6 +14095,8 @@ class Router: if messages is not None: counted_messages: Final = (system_message, *messages) if system_message is not None else messages return litellm.token_counter(messages=counted_messages, tools=tools) + if isinstance(input, list) and (embedding_tokens := _longest_embedding_input_tokens(input)) is not None: + return embedding_tokens if input is not None: from openai.types.responses.response_create_params import ResponseInputParam diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 2288d4311a9..c5a3df197ad 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -58,6 +58,7 @@ from litellm.router import ( _is_retriable_anthropic_status, _responses_stream_holds_event, _without_line_breaks, + _longest_embedding_input_tokens, Span, ) from litellm.router_strategy import simple_shuffle @@ -8020,6 +8021,62 @@ def test_count_pre_call_check_tokens_across_api_surfaces(): router._count_pre_call_check_tokens(messages=None, input=None) +def _embedding_router_with_context_window(max_input_tokens: int) -> Router: + return Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "fake-key"}, + "model_info": {"max_input_tokens": max_input_tokens}, + } + ], + enable_pre_call_checks=True, + ) + + +@pytest.mark.parametrize( + "embedding_input", + [ + pytest.param(["one two three four five six"] * 3, id="texts"), + pytest.param([list(range(6))] * 3, id="token-id-lists"), + ], +) +async def test_aembedding_batch_fits_when_every_input_fits_the_context_window(embedding_input): + router: Final = _embedding_router_with_context_window(10) + + response: Final = await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2]) + + assert response.data + + +@pytest.mark.parametrize( + "embedding_input", + [ + pytest.param(["one", " ".join(["word"] * 20)], id="texts"), + pytest.param(list(range(11)), id="token-ids"), + pytest.param([[1], list(range(11))], id="token-id-lists"), + ], +) +async def test_aembedding_rejects_a_batch_with_an_input_over_the_context_window(embedding_input): + router: Final = _embedding_router_with_context_window(10) + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2]) + +@pytest.mark.parametrize( + ("items", "expected"), + [ + pytest.param(["one", "one two three four"], litellm.token_counter(text="one two three four"), id="texts"), + pytest.param(list(range(7)), 7, id="token-ids"), + pytest.param([[1, 2], list(range(5))], 5, id="token-id-lists"), + pytest.param([1, "one"], None, id="mixed"), + pytest.param([{"type": "message", "role": "user", "content": "hi"}], None, id="responses-items"), + ], +) +def test_longest_embedding_input_tokens_counts_only_embedding_batches(items, expected): + assert _longest_embedding_input_tokens(items) == expected + + def test_count_pre_call_check_tokens_skips_the_history_a_compaction_item_replaces(): router = litellm.Router(model_list=[{"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}]) summary: Final = "The user planned a Tokyo trip."