mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): keep embedding images out of the pre-call count only on image backends, and skip tokenizing inputs that fit by bytes
Media leaves the embedding count only when every deployment in the group declares image embedding input, so a text-only backend still counts a data URI as text and an image backend outside the TPM media list no longer gets a false 400. A text no longer than the smallest window in UTF-8 bytes counts as its byte length without the tokenizer, since a token is never shorter than a byte.
This commit is contained in:
parent
94c7c8a325
commit
ed45d31192
3 changed files with 134 additions and 11 deletions
|
|
@ -666,14 +666,19 @@ def _is_pdf_block(block: Mapping[str, object]) -> bool:
|
|||
return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf")
|
||||
|
||||
|
||||
def _bedrock_base_model(resolved_model: str) -> str:
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
|
||||
return BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/"))
|
||||
|
||||
|
||||
def _bedrock_data_uri_media_scope(
|
||||
resolved_model: str, media: Sequence[Mapping[str, object]]
|
||||
) -> _DataUriMediaScope | None:
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.utils import supports_embedding_image_input
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/"))
|
||||
base_model: Final = _bedrock_base_model(resolved_model)
|
||||
if not supports_embedding_image_input(f"bedrock/{base_model}"):
|
||||
return None
|
||||
match BaseAWSLLM.get_bedrock_embedding_provider(base_model):
|
||||
|
|
@ -722,6 +727,32 @@ def split_embedding_input_tokens(
|
|||
return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape
|
||||
|
||||
|
||||
def _embeds_media(model: str) -> bool:
|
||||
from litellm.utils import supports_embedding_image_input
|
||||
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return False
|
||||
return supports_embedding_image_input(
|
||||
f"bedrock/{_bedrock_base_model(resolved_model)}" if provider == "bedrock" else model
|
||||
)
|
||||
|
||||
|
||||
def embedding_text_input(value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]]) -> _EmbeddingInputT:
|
||||
"""``value`` without its media when every backend in ``media_models`` declares image embedding
|
||||
input. A text-only backend reads a data URI as text, so for it the media stays in."""
|
||||
from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import split_embedding_media_for_tpm
|
||||
|
||||
text, media = split_embedding_media_for_tpm(value)
|
||||
if not media:
|
||||
return value
|
||||
models: Final = tuple(media_models())
|
||||
if not models or not all(map(_embeds_media, models)):
|
||||
return value
|
||||
return cast(_EmbeddingInputT, text) # cast-ok: the projection keeps the input's container shape
|
||||
|
||||
|
||||
def embedding_aware_token_count(
|
||||
messages: Sequence[Mapping[str, str]] | None,
|
||||
text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str]
|
||||
|
|
|
|||
|
|
@ -22,10 +22,10 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
|||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
embedding_text_input,
|
||||
offload_token_count,
|
||||
served_openai_models,
|
||||
single_encoding_openai_model,
|
||||
split_embedding_input_tokens,
|
||||
)
|
||||
from litellm.router_attempt import for_requested_model
|
||||
from litellm.router_python_host import python_host
|
||||
|
|
@ -59,11 +59,14 @@ def count_tokens(
|
|||
request_kwargs: Mapping[str, object] | None,
|
||||
tokenizer_model: str,
|
||||
deployments: Sequence[Mapping[str, object]] = (),
|
||||
fits_within: int = 0,
|
||||
) -> int:
|
||||
"""``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus
|
||||
the tools and Anthropic ``system`` block the message list never carries. Embedding media an
|
||||
item of ``deployments`` reads as media is left out of the input: it never counts as text. An
|
||||
embedding batch (a list of texts, token IDs or token ID lists) counts as its longest input."""
|
||||
the tools and Anthropic ``system`` block the message list never carries. Embedding media is left
|
||||
out of the input when every item of ``deployments`` declares image embedding input. An
|
||||
embedding batch (a list of texts, token IDs or token ID lists) counts as its longest input, and
|
||||
a text of at most ``fits_within`` UTF-8 bytes counts as its byte length without the tokenizer:
|
||||
a token is never shorter than a byte, so it fits any window of that size."""
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message
|
||||
|
||||
extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({})
|
||||
|
|
@ -81,8 +84,8 @@ def count_tokens(
|
|||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
||||
text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments))
|
||||
embedding_tokens: Final = _longest_embedding_input_tokens(host, text_input, tokenizer_model)
|
||||
text_input: Final = embedding_text_input(input, partial(deployment_backend_models, deployments))
|
||||
embedding_tokens: Final = _longest_embedding_input_tokens(host, text_input, tokenizer_model, fits_within)
|
||||
if embedding_tokens is not None:
|
||||
return embedding_tokens
|
||||
input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped
|
||||
|
|
@ -94,15 +97,18 @@ def count_tokens(
|
|||
raise ValueError("Either messages or input must be provided to count tokens")
|
||||
|
||||
|
||||
def _longest_embedding_input_tokens(host: "PythonHost", value: object, tokenizer_model: str) -> int | None:
|
||||
def _longest_embedding_input_tokens(
|
||||
host: "PythonHost", value: object, tokenizer_model: str, fits_within: int
|
||||
) -> int | None:
|
||||
"""An embedding model's context window bounds each input on its own, so a batch is never summed.
|
||||
None when ``value`` is not an embedding batch, which leaves it to the Responses input count."""
|
||||
items: Final = _items(value) if _is_list(value) else ()
|
||||
if not items:
|
||||
return None
|
||||
if all(isinstance(item, str) for item in items):
|
||||
texts: Final = tuple(item for item in items if isinstance(item, str))
|
||||
if len(texts) == len(items):
|
||||
counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # see count_tokens
|
||||
return max(counter(model=tokenizer_model, text=item) for item in items)
|
||||
return max(_text_tokens(counter, tokenizer_model, text, fits_within) for text in texts)
|
||||
if all(_is_token_id(item) for item in items):
|
||||
return _token_id_count(host, len(items))
|
||||
if all(_is_token_id_list(item) for item in items):
|
||||
|
|
@ -110,6 +116,11 @@ def _longest_embedding_input_tokens(host: "PythonHost", value: object, tokenizer
|
|||
return None
|
||||
|
||||
|
||||
def _text_tokens(counter: Callable[..., int], tokenizer_model: str, text: str, fits_within: int) -> int:
|
||||
size: Final = len(text.encode(errors="surrogatepass"))
|
||||
return size if size <= fits_within else counter(model=tokenizer_model, text=text)
|
||||
|
||||
|
||||
def _is_token_id(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool)
|
||||
|
||||
|
|
@ -279,6 +290,9 @@ class PreCallFacts:
|
|||
def _candidate_records(self) -> Sequence[Mapping[str, object]]:
|
||||
return tuple(self._record(id) for id in self._candidates)
|
||||
|
||||
def _fits_within(self) -> int:
|
||||
return min((limit for limit in self._windows.values() if limit is not None), default=0)
|
||||
|
||||
def _count(self) -> int | None:
|
||||
try:
|
||||
return count_tokens(
|
||||
|
|
@ -288,6 +302,7 @@ class PreCallFacts:
|
|||
self._request,
|
||||
self._tokenizer_model(),
|
||||
self._candidate_records(),
|
||||
self._fits_within(),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
|
||||
verbose_router_logger.error(
|
||||
|
|
@ -304,6 +319,7 @@ class PreCallFacts:
|
|||
self._request,
|
||||
self._tokenizer_model(),
|
||||
self._candidate_records(),
|
||||
self._fits_within(),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
|
||||
verbose_router_logger.error(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.router_core import RouterCore
|
||||
from litellm.router_python_host import Endpoint
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
from tests.unit.test_router.core.construction_precall_helpers import CallbackHost, restoring_model_cost, windowed
|
||||
from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, rust_core
|
||||
|
|
@ -55,6 +56,26 @@ UNSUPPORTED_MESSAGE: Final = (
|
|||
)
|
||||
|
||||
|
||||
class TokenizerHost(CallbackHost):
|
||||
"""Records each text the pre-call checks hand the tokenizer."""
|
||||
|
||||
__slots__ = ("tokenized",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tokenized: Final[list[object]] = []
|
||||
|
||||
def invoke(self, op: str) -> Endpoint:
|
||||
if op != "token_counter":
|
||||
return super().invoke(op)
|
||||
|
||||
def token_counter(**kwargs: object) -> int: # kwargs-ok: litellm.token_counter's arguments
|
||||
self.tokenized.append(kwargs.get("text"))
|
||||
return litellm.token_counter(**kwargs) # pyright: ignore[reportArgumentType]
|
||||
|
||||
return token_counter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def host() -> ScriptedHost:
|
||||
return CallbackHost()
|
||||
|
|
@ -279,6 +300,61 @@ async def test_an_embedding_image_stays_out_of_the_context_window_on_a_backend_t
|
|||
assert await _picked(core, times=2, input=EMBEDDING_IMAGE) == {"image"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
"bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",
|
||||
"oci/cohere.embed-english-image-v3.0",
|
||||
"empiriolabs/tongyi-embedding-vision-plus",
|
||||
"voyage/voyage-multimodal-3.5",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"embedding_input",
|
||||
[pytest.param(EMBEDDING_IMAGE, id="image"), pytest.param(["a red car", EMBEDDING_IMAGE], id="caption-and-image")],
|
||||
)
|
||||
async def test_an_embedding_image_stays_out_of_the_context_window_on_every_backend_declaring_image_input(
|
||||
make_core: MakeCore, backend: str, embedding_input: object
|
||||
) -> None:
|
||||
window: Final = litellm.get_model_info(backend)["max_input_tokens"]
|
||||
assert window is not None and litellm.token_counter(text=EMBEDDING_IMAGE) > window
|
||||
core: Final = make_core(model_list=[deployment("g", "image", model=backend)], enable_pre_call_checks=True)
|
||||
assert await _picked(core, times=2, input=embedding_input) == {"image"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_embedding_batch_tokenizes_only_the_inputs_longer_than_the_smallest_window_in_bytes() -> None:
|
||||
host: Final = TokenizerHost()
|
||||
core: Final = rust_core(
|
||||
host, model_list=[windowed("g", "small", 50), windowed("g", "big", 10000)], enable_pre_call_checks=True
|
||||
)
|
||||
at_the_window: Final = "word " * 10
|
||||
over_it_in_bytes_only: Final = "word " * 20
|
||||
assert len(at_the_window.encode()) == 50
|
||||
|
||||
picked: Final = await _picked(core, input=[at_the_window] * 2048 + [over_it_in_bytes_only])
|
||||
|
||||
assert picked == {"small", "big"}
|
||||
assert set(host.tokenized) == {over_it_in_bytes_only}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_embedding_input_that_fits_the_window_in_characters_but_not_in_tokens_is_counted(
|
||||
make_core: MakeCore,
|
||||
) -> None:
|
||||
dense: Final = "\ua66e" * 40
|
||||
assert len(dense) <= 50 < litellm.token_counter(text=dense)
|
||||
assert await _picked(_small_and_big(make_core, small=50), times=10, input=["hi", dense]) == {"big"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_embedding_input_with_a_lone_surrogate_is_counted(make_core: MakeCore) -> None:
|
||||
unpaired: Final = "\ud800" + "word " * 20
|
||||
assert litellm.token_counter(text=unpaired) > 10
|
||||
assert await _picked(_small_and_big(make_core, small=10), times=10, input=["hi", unpaired]) == {"big"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_embedding_image_counts_as_text_on_a_text_only_backend(make_core: MakeCore) -> None:
|
||||
core: Final = make_core(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue