diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e56dd83a277..c43b7870a03 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -666,14 +666,19 @@ def _is_pdf_block(block: Mapping[str, object]) -> bool: return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf") +def _bedrock_base_model(resolved_model: str) -> str: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + return BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/")) + + def _bedrock_data_uri_media_scope( resolved_model: str, media: Sequence[Mapping[str, object]] ) -> _DataUriMediaScope | None: from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM - from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.utils import supports_embedding_image_input - base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/")) + base_model: Final = _bedrock_base_model(resolved_model) if not supports_embedding_image_input(f"bedrock/{base_model}"): return None match BaseAWSLLM.get_bedrock_embedding_provider(base_model): @@ -722,6 +727,32 @@ def split_embedding_input_tokens( return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape +def _embeds_media(model: str) -> bool: + from litellm.utils import supports_embedding_image_input + + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model) + except litellm.exceptions.BadRequestError: + return False + return supports_embedding_image_input( + f"bedrock/{_bedrock_base_model(resolved_model)}" if provider == "bedrock" else model + ) + + +def embedding_text_input(value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]]) -> _EmbeddingInputT: + """``value`` without its media when every backend in ``media_models`` declares image embedding + input. A text-only backend reads a data URI as text, so for it the media stays in.""" + from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import split_embedding_media_for_tpm + + text, media = split_embedding_media_for_tpm(value) + if not media: + return value + models: Final = tuple(media_models()) + if not models or not all(map(_embeds_media, models)): + return value + return cast(_EmbeddingInputT, text) # cast-ok: the projection keeps the input's container shape + + def embedding_aware_token_count( messages: Sequence[Mapping[str, str]] | None, text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str] diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py index 97f420306e9..0981881f026 100644 --- a/litellm/router_rust/steps/precall.py +++ b/litellm/router_rust/steps/precall.py @@ -22,10 +22,10 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.token_counter import ( + embedding_text_input, offload_token_count, served_openai_models, single_encoding_openai_model, - split_embedding_input_tokens, ) from litellm.router_attempt import for_requested_model from litellm.router_python_host import python_host @@ -59,11 +59,14 @@ def count_tokens( request_kwargs: Mapping[str, object] | None, tokenizer_model: str, deployments: Sequence[Mapping[str, object]] = (), + fits_within: int = 0, ) -> int: """``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus - the tools and Anthropic ``system`` block the message list never carries. Embedding media an - item of ``deployments`` reads as media is left out of the input: it never counts as text. An - embedding batch (a list of texts, token IDs or token ID lists) counts as its longest input.""" + the tools and Anthropic ``system`` block the message list never carries. Embedding media is left + out of the input when every item of ``deployments`` declares image embedding input. An + embedding batch (a list of texts, token IDs or token ID lists) counts as its longest input, and + a text of at most ``fits_within`` UTF-8 bytes counts as its byte length without the tokenizer: + a token is never shorter than a byte, so it fits any window of that size.""" from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({}) @@ -81,8 +84,8 @@ def count_tokens( LiteLLMCompletionResponsesConfig, ) - text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) - embedding_tokens: Final = _longest_embedding_input_tokens(host, text_input, tokenizer_model) + text_input: Final = embedding_text_input(input, partial(deployment_backend_models, deployments)) + embedding_tokens: Final = _longest_embedding_input_tokens(host, text_input, tokenizer_model, fits_within) if embedding_tokens is not None: return embedding_tokens input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped @@ -94,15 +97,18 @@ def count_tokens( raise ValueError("Either messages or input must be provided to count tokens") -def _longest_embedding_input_tokens(host: "PythonHost", value: object, tokenizer_model: str) -> int | None: +def _longest_embedding_input_tokens( + host: "PythonHost", value: object, tokenizer_model: str, fits_within: int +) -> int | None: """An embedding model's context window bounds each input on its own, so a batch is never summed. None when ``value`` is not an embedding batch, which leaves it to the Responses input count.""" items: Final = _items(value) if _is_list(value) else () if not items: return None - if all(isinstance(item, str) for item in items): + texts: Final = tuple(item for item in items if isinstance(item, str)) + if len(texts) == len(items): counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # see count_tokens - return max(counter(model=tokenizer_model, text=item) for item in items) + return max(_text_tokens(counter, tokenizer_model, text, fits_within) for text in texts) if all(_is_token_id(item) for item in items): return _token_id_count(host, len(items)) if all(_is_token_id_list(item) for item in items): @@ -110,6 +116,11 @@ def _longest_embedding_input_tokens(host: "PythonHost", value: object, tokenizer return None +def _text_tokens(counter: Callable[..., int], tokenizer_model: str, text: str, fits_within: int) -> int: + size: Final = len(text.encode(errors="surrogatepass")) + return size if size <= fits_within else counter(model=tokenizer_model, text=text) + + def _is_token_id(value: object) -> bool: return isinstance(value, int) and not isinstance(value, bool) @@ -279,6 +290,9 @@ class PreCallFacts: def _candidate_records(self) -> Sequence[Mapping[str, object]]: return tuple(self._record(id) for id in self._candidates) + def _fits_within(self) -> int: + return min((limit for limit in self._windows.values() if limit is not None), default=0) + def _count(self) -> int | None: try: return count_tokens( @@ -288,6 +302,7 @@ class PreCallFacts: self._request, self._tokenizer_model(), self._candidate_records(), + self._fits_within(), ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( @@ -304,6 +319,7 @@ class PreCallFacts: self._request, self._tokenizer_model(), self._candidate_records(), + self._fits_within(), ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( diff --git a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py index 9cdb3f55d17..e5f8dd85063 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py @@ -18,6 +18,7 @@ import pytest import litellm from litellm.router_core import RouterCore +from litellm.router_python_host import Endpoint from litellm.types.router import RouterRateLimitError from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, restoring_model_cost, windowed from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rust_core @@ -55,6 +56,26 @@ UNSUPPORTED_MESSAGE: Final = ( ) +class TokenizerHost(CallbackHost): + """Records each text the pre-call checks hand the tokenizer.""" + + __slots__ = ("tokenized",) + + def __init__(self) -> None: + super().__init__() + self.tokenized: Final[list[object]] = [] + + def invoke(self, op: str) -> Endpoint: + if op != "token_counter": + return super().invoke(op) + + def token_counter(**kwargs: object) -> int: # kwargs-ok: litellm.token_counter's arguments + self.tokenized.append(kwargs.get("text")) + return litellm.token_counter(**kwargs) # pyright: ignore[reportArgumentType] + + return token_counter + + @pytest.fixture def host() -> ScriptedHost: return CallbackHost() @@ -279,6 +300,61 @@ async def test_an_embedding_image_stays_out_of_the_context_window_on_a_backend_t assert await _picked(core, times=2, input=EMBEDDING_IMAGE) == {"image"} +@pytest.mark.asyncio +@pytest.mark.parametrize( + "backend", + [ + "bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", + "oci/cohere.embed-english-image-v3.0", + "empiriolabs/tongyi-embedding-vision-plus", + "voyage/voyage-multimodal-3.5", + ], +) +@pytest.mark.parametrize( + "embedding_input", + [pytest.param(EMBEDDING_IMAGE, id="image"), pytest.param(["a red car", EMBEDDING_IMAGE], id="caption-and-image")], +) +async def test_an_embedding_image_stays_out_of_the_context_window_on_every_backend_declaring_image_input( + make_core: MakeCore, backend: str, embedding_input: object +) -> None: + window: Final = litellm.get_model_info(backend)["max_input_tokens"] + assert window is not None and litellm.token_counter(text=EMBEDDING_IMAGE) > window + core: Final = make_core(model_list=[deployment("g", "image", model=backend)], enable_pre_call_checks=True) + assert await _picked(core, times=2, input=embedding_input) == {"image"} + + +@pytest.mark.asyncio +async def test_an_embedding_batch_tokenizes_only_the_inputs_longer_than_the_smallest_window_in_bytes() -> None: + host: Final = TokenizerHost() + core: Final = rust_core( + host, model_list=[windowed("g", "small", 50), windowed("g", "big", 10000)], enable_pre_call_checks=True + ) + at_the_window: Final = "word " * 10 + over_it_in_bytes_only: Final = "word " * 20 + assert len(at_the_window.encode()) == 50 + + picked: Final = await _picked(core, input=[at_the_window] * 2048 + [over_it_in_bytes_only]) + + assert picked == {"small", "big"} + assert set(host.tokenized) == {over_it_in_bytes_only} + + +@pytest.mark.asyncio +async def test_an_embedding_input_that_fits_the_window_in_characters_but_not_in_tokens_is_counted( + make_core: MakeCore, +) -> None: + dense: Final = "\ua66e" * 40 + assert len(dense) <= 50 < litellm.token_counter(text=dense) + assert await _picked(_small_and_big(make_core, small=50), times=10, input=["hi", dense]) == {"big"} + + +@pytest.mark.asyncio +async def test_an_embedding_input_with_a_lone_surrogate_is_counted(make_core: MakeCore) -> None: + unpaired: Final = "\ud800" + "word " * 20 + assert litellm.token_counter(text=unpaired) > 10 + assert await _picked(_small_and_big(make_core, small=10), times=10, input=["hi", unpaired]) == {"big"} + + @pytest.mark.asyncio async def test_an_embedding_image_counts_as_text_on_a_text_only_backend(make_core: MakeCore) -> None: core: Final = make_core(