mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge 03b123ba20 into 4710c3b30e
This commit is contained in:
commit
f1aeecc246
2 changed files with 80 additions and 2 deletions
|
|
@ -46,7 +46,7 @@ import anyio
|
|||
import httpx
|
||||
import openai
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, StrictInt, StrictStr, TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never, overload
|
||||
|
||||
import litellm
|
||||
|
|
@ -604,12 +604,29 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
|
|||
return False
|
||||
|
||||
|
||||
def _longest_embedding_input_tokens(items: Sequence[object]) -> int | None:
|
||||
"""Token count of the longest input in an embedding batch, or None when `items` is not one.
|
||||
|
||||
An embedding model's context window bounds each input on its own, so the batch is never summed.
|
||||
"""
|
||||
if all(isinstance(item, int) for item in items):
|
||||
return len(items)
|
||||
try:
|
||||
batch: Final = _EMBEDDING_BATCH_ADAPTER.validate_python(items)
|
||||
except ValidationError:
|
||||
return None
|
||||
return max((litellm.token_counter(text=item) if isinstance(item, str) else len(item) for item in batch), default=0)
|
||||
|
||||
|
||||
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
|
||||
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
|
||||
_EMBEDDING_BATCH_ADAPTER: Final[TypeAdapter[tuple[StrictStr | tuple[StrictInt, ...], ...]]] = TypeAdapter(
|
||||
tuple[StrictStr | tuple[StrictInt, ...], ...]
|
||||
)
|
||||
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
|
||||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -14058,12 +14075,14 @@ class Router:
|
|||
Prompt content the message list never carries is read from `request_kwargs`:
|
||||
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
|
||||
Anthropic Messages top-level `system` block.
|
||||
|
||||
Embedding batches (`list[str]`, `list[int]`, `list[list[int]]`) count their longest input.
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.messages.utils import (
|
||||
anthropic_system_to_openai_message,
|
||||
)
|
||||
|
||||
extras: Final = request_kwargs if request_kwargs is not None else MappingProxyType({})
|
||||
extras: Final = request_kwargs if request_kwargs is not None else _NO_DEFAULT_PARAMS
|
||||
raw_instructions: Final = extras.get("instructions")
|
||||
instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None
|
||||
raw_tools: Final = extras.get("tools")
|
||||
|
|
@ -14076,6 +14095,8 @@ class Router:
|
|||
if messages is not None:
|
||||
counted_messages: Final = (system_message, *messages) if system_message is not None else messages
|
||||
return litellm.token_counter(messages=counted_messages, tools=tools)
|
||||
if isinstance(input, list) and (embedding_tokens := _longest_embedding_input_tokens(input)) is not None:
|
||||
return embedding_tokens
|
||||
if input is not None:
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ from litellm.router import (
|
|||
_is_retriable_anthropic_status,
|
||||
_responses_stream_holds_event,
|
||||
_without_line_breaks,
|
||||
_longest_embedding_input_tokens,
|
||||
Span,
|
||||
)
|
||||
from litellm.router_strategy import simple_shuffle
|
||||
|
|
@ -8020,6 +8021,62 @@ def test_count_pre_call_check_tokens_across_api_surfaces():
|
|||
router._count_pre_call_check_tokens(messages=None, input=None)
|
||||
|
||||
|
||||
def _embedding_router_with_context_window(max_input_tokens: int) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "embed",
|
||||
"litellm_params": {"model": "openai/text-embedding-3-small", "api_key": "fake-key"},
|
||||
"model_info": {"max_input_tokens": max_input_tokens},
|
||||
}
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"embedding_input",
|
||||
[
|
||||
pytest.param(["one two three four five six"] * 3, id="texts"),
|
||||
pytest.param([list(range(6))] * 3, id="token-id-lists"),
|
||||
],
|
||||
)
|
||||
async def test_aembedding_batch_fits_when_every_input_fits_the_context_window(embedding_input):
|
||||
router: Final = _embedding_router_with_context_window(10)
|
||||
|
||||
response: Final = await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2])
|
||||
|
||||
assert response.data
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"embedding_input",
|
||||
[
|
||||
pytest.param(["one", " ".join(["word"] * 20)], id="texts"),
|
||||
pytest.param(list(range(11)), id="token-ids"),
|
||||
pytest.param([[1], list(range(11))], id="token-id-lists"),
|
||||
],
|
||||
)
|
||||
async def test_aembedding_rejects_a_batch_with_an_input_over_the_context_window(embedding_input):
|
||||
router: Final = _embedding_router_with_context_window(10)
|
||||
|
||||
with pytest.raises(litellm.ContextWindowExceededError):
|
||||
await router.aembedding(model="embed", input=embedding_input, mock_response=[0.1, 0.2])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("items", "expected"),
|
||||
[
|
||||
pytest.param(["one", "one two three four"], litellm.token_counter(text="one two three four"), id="texts"),
|
||||
pytest.param(list(range(7)), 7, id="token-ids"),
|
||||
pytest.param([[1, 2], list(range(5))], 5, id="token-id-lists"),
|
||||
pytest.param([1, "one"], None, id="mixed"),
|
||||
pytest.param([{"type": "message", "role": "user", "content": "hi"}], None, id="responses-items"),
|
||||
],
|
||||
)
|
||||
def test_longest_embedding_input_tokens_counts_only_embedding_batches(items, expected):
|
||||
assert _longest_embedding_input_tokens(items) == expected
|
||||
|
||||
|
||||
def test_count_pre_call_check_tokens_skips_the_history_a_compaction_item_replaces():
|
||||
router = litellm.Router(model_list=[{"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}])
|
||||
summary: Final = "The user planned a Tokyo trip."
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue