This commit is contained in:
AdilBouWK 2026-10-24 19:17:50 +00:00 • committed by GitHub
commit f1aeecc246
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 80 additions and 2 deletions

View file

@ -46,7 +46,7 @@ import anyio
import httpx
import openai
from openai import AsyncOpenAI
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, StrictInt, StrictStr, TypeAdapter, ValidationError
from typing_extensions import assert_never, overload
import litellm
@ -604,12 +604,29 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
return False
def _longest_embedding_input_tokens(items: Sequence[object]) -> int | None:
"""Token count of the longest input in an embedding batch, or None when `items` is not one.
An embedding model's context window bounds each input on its own, so the batch is never summed.
"""
if all(isinstance(item, int) for item in items):
return len(items)
try:
batch: Final = _EMBEDDING_BATCH_ADAPTER.validate_python(items)
except ValidationError:
return None
return max((litellm.token_counter(text=item) if isinstance(item, str) else len(item) for item in batch), default=0)
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_EMBEDDING_BATCH_ADAPTER: Final[TypeAdapter[tuple[StrictStr | tuple[StrictInt, ...], ...]]] = TypeAdapter(
tuple[StrictStr | tuple[StrictInt, ...], ...]
)
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -14058,12 +14075,14 @@ class Router:
Prompt content the message list never carries is read from `request_kwargs`:
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
Anthropic Messages top-level `system` block.
Embedding batches (`list[str]`, `list[int]`, `list[list[int]]`) count their longest input.
"""
from litellm.llms.anthropic.pass_through.messages.utils import (
anthropic_system_to_openai_message,
)
extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({})
extras: Final = request_kwargs if request_kwargs is not None else _NO_DEFAULT_PARAMS
raw_instructions: Final = extras.get("instructions")
instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None
raw_tools: Final = extras.get("tools")
@ -14076,6 +14095,8 @@ class Router:
if messages is not None:
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
return litellm.token_counter(messages=counted_messages, tools=tools)
if isinstance(input, list) and (embedding_tokens := _longest_embedding_input_tokens(input)) is not None:
return embedding_tokens
if input is not None:
from openai.types.responses.response_create_params import ResponseInputParam

View file

@ -58,6 +58,7 @@ from litellm.router import (
_is_retriable_anthropic_status,
_responses_stream_holds_event,
_without_line_breaks,
_longest_embedding_input_tokens,
Span,
)
from litellm.router_strategy import simple_shuffle
@ -8020,6 +8021,62 @@ def test_count_pre_call_check_tokens_across_api_surfaces():
router._count_pre_call_check_tokens(messages=None, input=None)
def _embedding_router_with_context_window(max_input_tokens: int) -> Router:
return Router(
model_list=[
{
"model_name": "embed",
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "fake-key"},
"model_info": {"max_input_tokens": max_input_tokens},
}
],
enable_pre_call_checks=True,
)
@pytest.mark.parametrize(
"embedding_input",
[
pytest.param(["one two three four five six"] * 3, id="texts"),
pytest.param([list(range(6))] * 3, id="token-id-lists"),
],
)
async def test_aembedding_batch_fits_when_every_input_fits_the_context_window(embedding_input):
router: Final = _embedding_router_with_context_window(10)
response: Final = await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2])
assert response.data
@pytest.mark.parametrize(
"embedding_input",
[
pytest.param(["one", " ".join(["word"] * 20)], id="texts"),
pytest.param(list(range(11)), id="token-ids"),
pytest.param([[1], list(range(11))], id="token-id-lists"),
],
)
async def test_aembedding_rejects_a_batch_with_an_input_over_the_context_window(embedding_input):
router: Final = _embedding_router_with_context_window(10)
with pytest.raises(litellm.ContextWindowExceededError):
await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2])
@pytest.mark.parametrize(
("items", "expected"),
[
pytest.param(["one", "one two three four"], litellm.token_counter(text="one two three four"), id="texts"),
pytest.param(list(range(7)), 7, id="token-ids"),
pytest.param([[1, 2], list(range(5))], 5, id="token-id-lists"),
pytest.param([1, "one"], None, id="mixed"),
pytest.param([{"type": "message", "role": "user", "content": "hi"}], None, id="responses-items"),
],
)
def test_longest_embedding_input_tokens_counts_only_embedding_batches(items, expected):
assert _longest_embedding_input_tokens(items) == expected
def test_count_pre_call_check_tokens_skips_the_history_a_compaction_item_replaces():
router = litellm.Router(model_list=[{"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}])
summary: Final = "The user planned a Tokyo trip."