From c60a320b050d331320f2cf391728401ea154a3d3 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 16 Nov 2026 11:21:55 -0800 Subject: [PATCH] perf(token_counter): memoize token counts of long texts per tokenizer (#51904) * perf(token_counter): memoize token counts of long texts per tokenizer * test(prompt_caching): count a prompt no other test has counted in the loop lag checks * fix(token_counter): key the count memo on the string hash instead of blake2b * test(router): embed distinct texts so the batches are counted in full * test: time token counts over text no earlier count has memoized --- litellm/litellm_core_utils/token_counter.py | 66 +++++++++++-- .../test_a2a_streaming_iterator.py | 6 +- tests/unit/a2a_protocol/test_main.py | 6 +- .../caching/test_qdrant_semantic_cache.py | 4 +- .../unit/caching/test_redis_semantic_cache.py | 4 +- .../test_compression_interception_handler.py | 4 +- .../unit/litellm_core_utils/event_loop_lag.py | 8 ++ .../test_streaming_handler.py | 9 +- .../litellm_core_utils/test_token_counter.py | 94 +++++++++++++++++-- .../context_management/test_compact.py | 4 +- .../context_management/test_dispatcher.py | 4 +- tests/unit/proxy/hooks/test_tpm_concurrent.py | 4 +- .../test_streaming_handler.py | 8 +- ...test_proxy_server_endpoints_and_startup.py | 6 +- ..._utils_model_creation_and_error_logging.py | 4 +- .../test_prompt_caching_deployment_check.py | 15 ++- tests/unit/test_router/test_router.py | 7 +- 17 files changed, 202 insertions(+), 51 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f51050114fe..d0a5a8f0fad 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,7 +5,9 @@ import io import math import re import struct +import threading from bisect import bisect_right +from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import cached_property @@ -1233,19 +1235,67 @@ def _evenly_spaced_samples(text: str, total_chars: int) -> tuple[str, ...]: ) +_MemoKey: TypeAlias = tuple[int, int, int] +_MemoEntry: TypeAlias = tuple[object, int] + + +class TokenCountMemo: + """Least recently used token counts of long texts, so a conversation resent with one more turn only tokenizes + the new turn. Entries are keyed by the tokenizer's id and the text's length and hash, never the text itself, and + hold their tokenizer so that id cannot be reused by another one while the entry lives.""" + + __slots__ = ("_counts", "_lock", "_max_entries", "_min_chars") + + def __init__(self, max_entries: int = 16_384, min_chars: int = 1024) -> None: + self._counts: Final[OrderedDict[_MemoKey, _MemoEntry]] = OrderedDict() # mutable-ok: an LRU evicts in place + self._lock: Final = threading.Lock() + self._max_entries: Final = max_entries + self._min_chars: Final = min_chars + + def wrap(self, tokenizer: object, count: TokenCounterFunction) -> TokenCounterFunction: + def count_tokens(text: str) -> int: + if len(text) < self._min_chars: + return count(text) + key: Final = (id(tokenizer), len(text), hash(text)) + with self._lock: + cached: Final = self._counts.get(key) + if cached is not None: + self._counts.move_to_end(key) + return cached[1] + counted: Final = count(text) + with self._lock: + self._counts[key] = (tokenizer, counted) + self._counts.move_to_end(key) + if len(self._counts) > self._max_entries: + self._counts.popitem(last=False) + return counted + + return count_tokens + + +_TOKEN_COUNT_MEMO: Final = TokenCountMemo() + + def _get_count_function( model: str | None, custom_tokenizer: dict | SelectTokenizerResponse | None = None, ) -> TokenCounterFunction: - return _get_extrapolating_count_function(_get_exact_count_function(model, custom_tokenizer)) + exact: Final = _resolve_exact_count(model, custom_tokenizer) + count: Final = _get_extrapolating_count_function(exact.count) + return count if custom_tokenizer is not None else _TOKEN_COUNT_MEMO.wrap(exact.tokenizer, count) -def _get_exact_count_function( +@dataclass(frozen=True, slots=True) +class _ExactCount: + tokenizer: "Encoding | HuggingFace" + count: TokenCounterFunction + + +def _resolve_exact_count( model: str | None, custom_tokenizer: dict | SelectTokenizerResponse | None = None, -) -> TokenCounterFunction: - """ - Get the function to count tokens based on the model and custom tokenizer.""" +) -> _ExactCount: + """The tokenizer and the exact count function for the model and custom tokenizer.""" from litellm.utils import select_tokenizer if model is not None or custom_tokenizer is not None: @@ -1256,7 +1306,7 @@ def _get_exact_count_function( def encode_length(text: str) -> int: return _encoding_count(default_encoding, text) - return _get_tiktoken_count_function(encode_length) + return _ExactCount(default_encoding, _get_tiktoken_count_function(encode_length)) if tokenizer_json["type"] == "huggingface_tokenizer": tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"] @@ -1265,14 +1315,14 @@ def _get_exact_count_function( return tokenizer.count(text) return len(tokenizer.encode_batch_fast([text])[0]) - return count_tokens + return _ExactCount(tokenizer, count_tokens) if tokenizer_json["type"] == "openai_tokenizer": encoding: Final = openai_tokenizer_encoding(model or "") def encode_length(text: str) -> int: return _encoding_count(encoding, text) - return _get_tiktoken_count_function(encode_length) + return _ExactCount(encoding, _get_tiktoken_count_function(encode_length)) raise ValueError("Unsupported tokenizer type") diff --git a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py index 9e78ba7a077..ae673e50c80 100644 --- a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py +++ b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py @@ -100,10 +100,10 @@ class _AgentChunk: @pytest.mark.asyncio async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -121,12 +121,12 @@ async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch): ) async def _stream(): - yield _AgentChunk(text * 100) + yield _AgentChunk(uncounted_text()) iterator = A2AStreamingIterator( stream=_stream(), request=SimpleNamespace( - params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": text * 100}]}) + params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]}) ), logging_obj=logging_obj, agent_name="test-agent", diff --git a/tests/unit/a2a_protocol/test_main.py b/tests/unit/a2a_protocol/test_main.py index c0cabee1127..4d06f79b991 100644 --- a/tests/unit/a2a_protocol/test_main.py +++ b/tests/unit/a2a_protocol/test_main.py @@ -729,10 +729,10 @@ async def test_streamed_send_is_timed_from_before_the_agent_card_fetch( @pytest.mark.asyncio async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -745,11 +745,11 @@ async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch): reply = _conv.pb2_v10.StreamResponse() reply.message.message_id = "reply-1" reply.message.role = _conv.pb2_v10.Role.ROLE_AGENT - reply.message.parts.add().text = text * 100 + reply.message.parts.add().text = uncounted_text() request = SendMessageRequest( id="r1", params=MessageSendParams( - message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": text * 100}]} + message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]} ), ) diff --git a/tests/unit/caching/test_qdrant_semantic_cache.py b/tests/unit/caching/test_qdrant_semantic_cache.py index 9d1dc384d92..ff711e66407 100644 --- a/tests/unit/caching/test_qdrant_semantic_cache.py +++ b/tests/unit/caching/test_qdrant_semantic_cache.py @@ -995,10 +995,10 @@ def test_qdrant_semantic_cache_defaults_embedding_timeout(): @pytest.mark.asyncio async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -1019,7 +1019,7 @@ async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch): _router_proxy_module(router, "sem-embed"), ) - response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100)) + response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text())) assert response["data"][0]["embedding"] == [0.1, 0.2] assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5 diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 9b6f6de0d7d..22b825e6987 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -1430,10 +1430,10 @@ def test_redis_semantic_cache_defaults_embedding_timeout(): @pytest.mark.asyncio async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -1450,7 +1450,7 @@ async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch): router.aembedding = AsyncMock(return_value={"data": [{"embedding": [0.1, 0.2]}]}) _proxy_with_router(monkeypatch, router, "sem-embed") - embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100)) + embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text())) assert embedding == [0.1, 0.2] assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5 diff --git a/tests/unit/integrations/compression_interception/test_compression_interception_handler.py b/tests/unit/integrations/compression_interception/test_compression_interception_handler.py index 549c0363942..9c5ef7ac4e4 100644 --- a/tests/unit/integrations/compression_interception/test_compression_interception_handler.py +++ b/tests/unit/integrations/compression_interception/test_compression_interception_handler.py @@ -527,17 +527,17 @@ async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch): @pytest.mark.asyncio async def test_pre_call_hook_counts_tokens_off_the_event_loop(): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) model = "anthropic/claude-fable-5" warm_tokenizer(model) logger = CompressionInterceptionLogger(compression_trigger=10_000_000) - messages = [{"role": "user", "content": text * 100}] + messages = [{"role": "user", "content": uncounted_text()}] kwargs = {"model": model, "messages": messages} result, took, lags = await timed_with_loop_lags( diff --git a/tests/unit/litellm_core_utils/event_loop_lag.py b/tests/unit/litellm_core_utils/event_loop_lag.py index 5697b3203f5..3b13497c3c2 100644 --- a/tests/unit/litellm_core_utils/event_loop_lag.py +++ b/tests/unit/litellm_core_utils/event_loop_lag.py @@ -1,14 +1,22 @@ import asyncio import gc import time +import uuid from collections.abc import Awaitable, Callable from typing import Final, TypeVar import litellm +from tests.large_text import text T = TypeVar("T") +def uncounted_text() -> str: + """`tests.large_text.text` 100 times behind a fresh prefix, so no earlier count can have cached its token count and + a timed count of it is a full tokenization.""" + return f"{uuid.uuid4()} {text * 100}" + + def warm_tokenizer(model: str) -> None: litellm.token_counter(model=model, text="load the tokenizer before anything is timed") diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index a44fa823cce..c96cf3cf73d 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -9497,17 +9497,18 @@ class TestContinuationDisqualifiers: @pytest.mark.asyncio async def test_async_stream_without_usage_counts_tokens_off_the_event_loop(): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) model = "gpt-5.6-luna" warm_tokenizer(model) - messages = [{"role": "user", "content": text * 100}] - content_chunks = [_make_chunk(text) for _ in range(100)] + messages = [{"role": "user", "content": uncounted_text()}] + completion = uncounted_text() + content_chunks = [_make_chunk(completion[start : start + 25_000]) for start in range(0, len(completion), 25_000)] stop_chunk = ModelResponseStream( id="test", created=1741037890, @@ -9536,7 +9537,7 @@ async def test_async_stream_without_usage_counts_tokens_off_the_event_loop(): chunks, took, lags = await timed_with_loop_lags(consume) - assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == text * 100 + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == completion assert chunks[-1].usage.prompt_tokens > 100_000 assert chunks[-1].usage.completion_tokens > 100_000 assert_loop_stayed_free(took, lags) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index c7bf2794366..ae356219620 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -35,12 +35,13 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( + TokenCountMemo, UnsupportedContentTypeError, _count_content_list, _encoding_count, - _get_exact_count_function, _get_extrapolating_count_function, _get_tiktoken_count_function, + _resolve_exact_count, calculate_img_tokens, estimate_image_tokens, has_only_text_content, @@ -57,6 +58,7 @@ from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) from tests.unit.litellm_core_utils.messages_with_counts import ( @@ -175,7 +177,7 @@ async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free() warm_tokenizer("claude-fable-5") tokens, took, lags = await timed_with_loop_lags( - lambda: asyncify(token_counter_new)(model="claude-fable-5", text=text * 100) + lambda: asyncify(token_counter_new)(model="claude-fable-5", text=uncounted_text()) ) assert tokens > 0 @@ -220,7 +222,7 @@ class _SlowEncoder: @pytest.mark.asyncio async def test_offloaded_counts_do_not_borrow_from_the_shared_thread_pool(): encoder: Final = _SlowEncoder() - count: Final = _get_exact_count_function(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}) + count: Final = _resolve_exact_count(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}).count shared_pool: Final = anyio.to_thread.current_default_thread_limiter() burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_COUNTS @@ -279,7 +281,7 @@ def test_token_counter_applies_the_default_cap(): max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars] over_the_cap: Final = prose + "a" * 200_000 - exact: Final = _get_exact_count_function("gpt-5.6")(over_the_cap) + exact: Final = _resolve_exact_count("gpt-5.6").count(over_the_cap) estimate: Final = token_counter_new(model="gpt-5.6", text=over_the_cap) @@ -3534,8 +3536,8 @@ def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encodin def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: text_value: Final = "A tokenizer fallback should preserve the model encoding." - custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value) - model_count: Final = _get_exact_count_function("gpt-3.5-turbo", None)(text_value) + custom_count: Final = _resolve_exact_count("gpt-3.5-turbo", {}).count(text_value) + model_count: Final = _resolve_exact_count("gpt-3.5-turbo", None).count(text_value) assert custom_count == model_count @@ -3870,3 +3872,83 @@ def test_openai_model_check_holds_while_openai_models_are_registered_concurrentl ) def test_has_only_text_content_accepts_only_plain_text_chat_messages(messages: list[object], text_only: bool) -> None: assert has_only_text_content(messages) is text_only + + +_MEMOIZED_TEXT: Final = "a long turn of the conversation, resent with every request. " * 40 + + +class _CallCountingCounter: + def __init__(self) -> None: + self.calls = 0 + + def __call__(self, text: str) -> int: + self.calls += 1 + return len(text) // 4 + + +class _CallCountingEncoder: + def __init__(self) -> None: + self.calls = 0 + + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: + self.calls += 1 + return [[0] * len(text) for text in texts] + + +def test_token_count_memo_counts_a_repeated_long_text_once(): + counter: Final = _CallCountingCounter() + count: Final = TokenCountMemo().wrap(object(), counter) + + assert count(_MEMOIZED_TEXT) == count(_MEMOIZED_TEXT) == len(_MEMOIZED_TEXT) // 4 + assert counter.calls == 1 + + +def test_token_count_memo_counts_text_below_the_threshold_every_time(): + counter: Final = _CallCountingCounter() + count: Final = TokenCountMemo(min_chars=1024).wrap(object(), counter) + short_text: Final = _MEMOIZED_TEXT[:1023] + + assert count(short_text) == count(short_text) + assert counter.calls == 2 + + +def test_token_count_memo_evicts_the_least_recently_used_count(): + counter: Final = _CallCountingCounter() + count: Final = TokenCountMemo(max_entries=2).wrap(object(), counter) + first, second, third = (f"{index} {_MEMOIZED_TEXT}" for index in range(3)) + + tuple(map(count, (first, second, first, third))) + assert counter.calls == 3 + count(first) + assert counter.calls == 3 + count(second) + assert counter.calls == 4 + + +def test_token_count_memo_holds_no_reference_to_the_text(): + memo: Final = TokenCountMemo() + text: Final = "".join(("fresh ", _MEMOIZED_TEXT)) + references_before: Final = sys.getrefcount(text) + + memo.wrap(object(), len)(text) + + assert sys.getrefcount(text) == references_before + + +def test_memoized_counts_match_uncached_counts_for_each_model_sharing_the_memo(): + text: Final = "Ünïcödé prose, code() and 🙂 emoji in one long message, counted for several models. " * 60 + models: Final = ("gpt-4o", "gpt-3.5-turbo", "claude-sonnet-4-5") + uncached: Final = tuple(_resolve_exact_count(model).count(text) for model in models) + + assert len(set(uncached)) == len(models) + assert tuple(token_counter_new(model=model, text=text) for model in models * 2) == uncached * 2 + + +def test_custom_tokenizer_counts_are_not_memoized(): + encoder: Final = _CallCountingEncoder() + custom_tokenizer: Final = {"type": "huggingface_tokenizer", "tokenizer": encoder} + + counts: Final = tuple(token_counter_new(custom_tokenizer=custom_tokenizer, text=_MEMOIZED_TEXT) for _ in range(2)) + + assert counts == (len(_MEMOIZED_TEXT), len(_MEMOIZED_TEXT)) + assert encoder.calls == 2 diff --git a/tests/unit/llms/anthropic/messages/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py index 32899a8e3c8..efd2b4db8c9 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -2814,10 +2814,10 @@ def test_build_summary_messages_keeps_midturn_system_correction_in_place(): async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -2828,7 +2828,7 @@ async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch): monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "claude-haiku-4-5") warm_tokenizer(MODEL) - messages = [{"role": "user", "content": text * 100}, *_simple_messages()] + messages = [{"role": "user", "content": uncounted_text()}, *_simple_messages()] result, took, lags = await timed_with_loop_lags( lambda: apply_compact_20260112( model=MODEL, diff --git a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index e55af602013..1c9fc4d9483 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -131,15 +131,15 @@ async def test_malformed_edit_entries_are_skipped(): async def test_sync_editor_counts_tokens_off_the_event_loop(): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) warm_tokenizer(MODEL) - messages = [{"role": "user", "content": text * 100}, *_history_with_two_tool_pairs()] + messages = [{"role": "user", "content": uncounted_text()}, *_history_with_two_tool_pairs()] result, took, lags = await timed_with_loop_lags( lambda: apply_context_management( diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index d60b2a320d7..3ca39a847d4 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -3404,10 +3404,10 @@ async def test_post_call_success_hook_contains_header_merge_failures(rate_limite @pytest.mark.asyncio async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loop(rate_limiter): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -3416,7 +3416,7 @@ async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loo warm_tokenizer("claude-fable-5") data: dict[str, object] = { "model": "claude-fable-5", - "messages": [{"role": "user", "content": text * 100}], + "messages": [{"role": "user", "content": uncounted_text()}], } itpm_descriptor = { "key": PROJECT_ITPM_DESCRIPTOR_KEY, diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index bf2cf8f1120..8da5a484d36 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -165,10 +165,10 @@ def _interrupted_anthropic_stream(model: str, output_text: str) -> list[bytes]: async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(caplog): from unittest.mock import AsyncMock - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -190,7 +190,7 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event request_body={"model": model, "stream": True}, endpoint_type=EndpointType.ANTHROPIC, start_time=datetime.now(), - raw_bytes=_interrupted_anthropic_stream(model, text * 100), + raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()), end_time=datetime.now(), model=model, ) @@ -206,10 +206,10 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(caplog): from unittest.mock import AsyncMock - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -232,7 +232,7 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop( litellm_logging_obj=logging_obj, endpoint_type=EndpointType.ANTHROPIC, request_body={"model": model, "stream": True}, - raw_bytes=_interrupted_anthropic_stream(model, text * 100), + raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()), exception=RuntimeError("upstream closed the stream"), ) ) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index bdb9c5bbd66..5481a17d071 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -15711,10 +15711,10 @@ def test_settings_store_exposes_dashboard_saved_mcp_client_allowlist_to_the_mcp_ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_count(monkeypatch): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -15722,7 +15722,9 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun warm_tokenizer("claude-fable-5") response, took, lags = await timed_with_loop_lags( - lambda: proxy_server_module.count_request_tokens(TokenCountRequest(model="claude-fable-5", prompt=text * 100)) + lambda: proxy_server_module.count_request_tokens( + TokenCountRequest(model="claude-fable-5", prompt=uncounted_text()) + ) ) assert response.total_tokens > 0 diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index f4879bc9565..2c5560c15de 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -4056,10 +4056,10 @@ def test_a_dispatched_failure_lifts_the_fields_the_spend_log_needs(): async def test_a_dispatched_failure_is_counted_off_the_event_loop(): from unittest.mock import AsyncMock, patch - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -4070,7 +4070,7 @@ async def test_a_dispatched_failure_is_counted_off_the_event_loop(): "first_api_call_start_time": 1700000000.0, "call_type": "acompletion", "model": "claude-fable-5", - "messages": [{"role": "user", "content": text * 100}], + "messages": [{"role": "user", "content": uncounted_text()}], } ), "metadata": {}, diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 76e9a70614f..f6e35d26666 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -766,17 +766,17 @@ async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost @pytest.mark.asyncio async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop(): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) warm_tokenizer("anthropic/claude-fable-5") check = PromptCachingDeploymentCheck(cache=DualCache()) deployments = _deployments("anthropic/claude-fable-5", "anthropic/claude-fable-5") - messages = cast(list[AllMessageValues], [{"role": "user", "content": text * 100}]) + messages = cast(list[AllMessageValues], [{"role": "user", "content": uncounted_text()}]) result, took, lags = await timed_with_loop_lags( lambda: check.async_filter_deployments( @@ -790,10 +790,10 @@ async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop(): @pytest.mark.asyncio async def test_async_log_success_event_counts_the_prompt_off_the_event_loop(): - from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, timed_with_loop_lags, + uncounted_text, warm_tokenizer, ) @@ -802,7 +802,14 @@ async def test_async_log_success_event_counts_the_prompt_off_the_event_loop(): check = PromptCachingDeploymentCheck(cache=cache) messages = cast( list[AllMessageValues], - [{"role": "user", "content": [{"type": "text", "text": text * 100, "cache_control": {"type": "ephemeral"}}]}], + [ + { + "role": "user", + "content": [ + {"type": "text", "text": uncounted_text(), "cache_control": {"type": "ephemeral"}} + ], + } + ], ) standard_logging_object = { "call_type": "acompletion", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5824b233126..c8f9ea6b3a3 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -4699,10 +4699,11 @@ async def test_a_chat_pre_call_count_does_not_wait_behind_long_embedding_batches ], enable_pre_call_checks=True, ) - batch: Final = ["word " * 2100] * 76 batches: Final = tuple( - asyncio.create_task(router.aembedding(model="embed", input=batch)) - for _ in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) + asyncio.create_task( + router.aembedding(model="embed", input=[f"{batch} {text} " + "word " * 2100 for text in range(76)]) + ) + for batch in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) ) await asyncio.wait_for(_until_every_count_slot_is_taken(), timeout=30) chat: Final = asyncio.create_task(router.acompletion(model="chat", messages=[{"role": "user", "content": "hi"}]))