mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
perf(token_counter): memoize token counts of long texts per tokenizer (#51904)
* perf(token_counter): memoize token counts of long texts per tokenizer * test(prompt_caching): count a prompt no other test has counted in the loop lag checks * fix(token_counter): key the count memo on the string hash instead of blake2b * test(router): embed distinct texts so the batches are counted in full * test: time token counts over text no earlier count has memoized
This commit is contained in:
parent
e4759def40
commit
c60a320b05
17 changed files with 202 additions and 51 deletions
|
|
@ -5,7 +5,9 @@ import io
|
|||
import math
|
||||
import re
|
||||
import struct
|
||||
import threading
|
||||
from bisect import bisect_right
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
|
|
@ -1233,19 +1235,67 @@ def _evenly_spaced_samples(text: str, total_chars: int) -> tuple[str, ...]:
|
|||
)
|
||||
|
||||
|
||||
_MemoKey: TypeAlias = tuple[int, int, int]
|
||||
_MemoEntry: TypeAlias = tuple[object, int]
|
||||
|
||||
|
||||
class TokenCountMemo:
|
||||
"""Least recently used token counts of long texts, so a conversation resent with one more turn only tokenizes
|
||||
the new turn. Entries are keyed by the tokenizer's id and the text's length and hash, never the text itself, and
|
||||
hold their tokenizer so that id cannot be reused by another one while the entry lives."""
|
||||
|
||||
__slots__ = ("_counts", "_lock", "_max_entries", "_min_chars")
|
||||
|
||||
def __init__(self, max_entries: int = 16_384, min_chars: int = 1024) -> None:
|
||||
self._counts: Final[OrderedDict[_MemoKey, _MemoEntry]] = OrderedDict() # mutable-ok: an LRU evicts in place
|
||||
self._lock: Final = threading.Lock()
|
||||
self._max_entries: Final = max_entries
|
||||
self._min_chars: Final = min_chars
|
||||
|
||||
def wrap(self, tokenizer: object, count: TokenCounterFunction) -> TokenCounterFunction:
|
||||
def count_tokens(text: str) -> int:
|
||||
if len(text) < self._min_chars:
|
||||
return count(text)
|
||||
key: Final = (id(tokenizer), len(text), hash(text))
|
||||
with self._lock:
|
||||
cached: Final = self._counts.get(key)
|
||||
if cached is not None:
|
||||
self._counts.move_to_end(key)
|
||||
return cached[1]
|
||||
counted: Final = count(text)
|
||||
with self._lock:
|
||||
self._counts[key] = (tokenizer, counted)
|
||||
self._counts.move_to_end(key)
|
||||
if len(self._counts) > self._max_entries:
|
||||
self._counts.popitem(last=False)
|
||||
return counted
|
||||
|
||||
return count_tokens
|
||||
|
||||
|
||||
_TOKEN_COUNT_MEMO: Final = TokenCountMemo()
|
||||
|
||||
|
||||
def _get_count_function(
|
||||
model: str | None,
|
||||
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
|
||||
) -> TokenCounterFunction:
|
||||
return _get_extrapolating_count_function(_get_exact_count_function(model, custom_tokenizer))
|
||||
exact: Final = _resolve_exact_count(model, custom_tokenizer)
|
||||
count: Final = _get_extrapolating_count_function(exact.count)
|
||||
return count if custom_tokenizer is not None else _TOKEN_COUNT_MEMO.wrap(exact.tokenizer, count)
|
||||
|
||||
|
||||
def _get_exact_count_function(
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ExactCount:
|
||||
tokenizer: "Encoding | HuggingFace"
|
||||
count: TokenCounterFunction
|
||||
|
||||
|
||||
def _resolve_exact_count(
|
||||
model: str | None,
|
||||
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
|
||||
) -> TokenCounterFunction:
|
||||
"""
|
||||
Get the function to count tokens based on the model and custom tokenizer."""
|
||||
) -> _ExactCount:
|
||||
"""The tokenizer and the exact count function for the model and custom tokenizer."""
|
||||
from litellm.utils import select_tokenizer
|
||||
|
||||
if model is not None or custom_tokenizer is not None:
|
||||
|
|
@ -1256,7 +1306,7 @@ def _get_exact_count_function(
|
|||
def encode_length(text: str) -> int:
|
||||
return _encoding_count(default_encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
return _ExactCount(default_encoding, _get_tiktoken_count_function(encode_length))
|
||||
if tokenizer_json["type"] == "huggingface_tokenizer":
|
||||
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
|
||||
|
||||
|
|
@ -1265,14 +1315,14 @@ def _get_exact_count_function(
|
|||
return tokenizer.count(text)
|
||||
return len(tokenizer.encode_batch_fast([text])[0])
|
||||
|
||||
return count_tokens
|
||||
return _ExactCount(tokenizer, count_tokens)
|
||||
if tokenizer_json["type"] == "openai_tokenizer":
|
||||
encoding: Final = openai_tokenizer_encoding(model or "")
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return _encoding_count(encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
return _ExactCount(encoding, _get_tiktoken_count_function(encode_length))
|
||||
raise ValueError("Unsupported tokenizer type")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -100,10 +100,10 @@ class _AgentChunk:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -121,12 +121,12 @@ async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
|
|||
)
|
||||
|
||||
async def _stream():
|
||||
yield _AgentChunk(text * 100)
|
||||
yield _AgentChunk(uncounted_text())
|
||||
|
||||
iterator = A2AStreamingIterator(
|
||||
stream=_stream(),
|
||||
request=SimpleNamespace(
|
||||
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": text * 100}]})
|
||||
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]})
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
agent_name="test-agent",
|
||||
|
|
|
|||
|
|
@ -729,10 +729,10 @@ async def test_streamed_send_is_timed_from_before_the_agent_card_fetch(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -745,11 +745,11 @@ async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch):
|
|||
reply = _conv.pb2_v10.StreamResponse()
|
||||
reply.message.message_id = "reply-1"
|
||||
reply.message.role = _conv.pb2_v10.Role.ROLE_AGENT
|
||||
reply.message.parts.add().text = text * 100
|
||||
reply.message.parts.add().text = uncounted_text()
|
||||
request = SendMessageRequest(
|
||||
id="r1",
|
||||
params=MessageSendParams(
|
||||
message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": text * 100}]}
|
||||
message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]}
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -995,10 +995,10 @@ def test_qdrant_semantic_cache_defaults_embedding_timeout():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -1019,7 +1019,7 @@ async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch):
|
|||
_router_proxy_module(router, "sem-embed"),
|
||||
)
|
||||
|
||||
response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
|
||||
response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text()))
|
||||
|
||||
assert response["data"][0]["embedding"] == [0.1, 0.2]
|
||||
assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5
|
||||
|
|
|
|||
|
|
@ -1430,10 +1430,10 @@ def test_redis_semantic_cache_defaults_embedding_timeout():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -1450,7 +1450,7 @@ async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch):
|
|||
router.aembedding = AsyncMock(return_value={"data": [{"embedding": [0.1, 0.2]}]})
|
||||
_proxy_with_router(monkeypatch, router, "sem-embed")
|
||||
|
||||
embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
|
||||
embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text()))
|
||||
|
||||
assert embedding == [0.1, 0.2]
|
||||
assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5
|
||||
|
|
|
|||
|
|
@ -527,17 +527,17 @@ async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_call_hook_counts_tokens_off_the_event_loop():
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
model = "anthropic/claude-fable-5"
|
||||
warm_tokenizer(model)
|
||||
logger = CompressionInterceptionLogger(compression_trigger=10_000_000)
|
||||
messages = [{"role": "user", "content": text * 100}]
|
||||
messages = [{"role": "user", "content": uncounted_text()}]
|
||||
kwargs = {"model": model, "messages": messages}
|
||||
|
||||
result, took, lags = await timed_with_loop_lags(
|
||||
|
|
|
|||
|
|
@ -1,14 +1,22 @@
|
|||
import asyncio
|
||||
import gc
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import litellm
|
||||
from tests.large_text import text
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def uncounted_text() -> str:
|
||||
"""`tests.large_text.text` 100 times behind a fresh prefix, so no earlier count can have cached its token count and
|
||||
a timed count of it is a full tokenization."""
|
||||
return f"{uuid.uuid4()} {text * 100}"
|
||||
|
||||
|
||||
def warm_tokenizer(model: str) -> None:
|
||||
litellm.token_counter(model=model, text="load the tokenizer before anything is timed")
|
||||
|
||||
|
|
|
|||
|
|
@ -9497,17 +9497,18 @@ class TestContinuationDisqualifiers:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
model = "gpt-5.6-luna"
|
||||
warm_tokenizer(model)
|
||||
messages = [{"role": "user", "content": text * 100}]
|
||||
content_chunks = [_make_chunk(text) for _ in range(100)]
|
||||
messages = [{"role": "user", "content": uncounted_text()}]
|
||||
completion = uncounted_text()
|
||||
content_chunks = [_make_chunk(completion[start : start + 25_000]) for start in range(0, len(completion), 25_000)]
|
||||
stop_chunk = ModelResponseStream(
|
||||
id="test",
|
||||
created=1741037890,
|
||||
|
|
@ -9536,7 +9537,7 @@ async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
|
|||
|
||||
chunks, took, lags = await timed_with_loop_lags(consume)
|
||||
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == text * 100
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == completion
|
||||
assert chunks[-1].usage.prompt_tokens > 100_000
|
||||
assert chunks[-1].usage.completion_tokens > 100_000
|
||||
assert_loop_stayed_free(took, lags)
|
||||
|
|
|
|||
|
|
@ -35,12 +35,13 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
TokenCountMemo,
|
||||
UnsupportedContentTypeError,
|
||||
_count_content_list,
|
||||
_encoding_count,
|
||||
_get_exact_count_function,
|
||||
_get_extrapolating_count_function,
|
||||
_get_tiktoken_count_function,
|
||||
_resolve_exact_count,
|
||||
calculate_img_tokens,
|
||||
estimate_image_tokens,
|
||||
has_only_text_content,
|
||||
|
|
@ -57,6 +58,7 @@ from tests.large_text import text
|
|||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
from tests.unit.litellm_core_utils.messages_with_counts import (
|
||||
|
|
@ -175,7 +177,7 @@ async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free()
|
|||
warm_tokenizer("claude-fable-5")
|
||||
|
||||
tokens, took, lags = await timed_with_loop_lags(
|
||||
lambda: asyncify(token_counter_new)(model="claude-fable-5", text=text * 100)
|
||||
lambda: asyncify(token_counter_new)(model="claude-fable-5", text=uncounted_text())
|
||||
)
|
||||
|
||||
assert tokens > 0
|
||||
|
|
@ -220,7 +222,7 @@ class _SlowEncoder:
|
|||
@pytest.mark.asyncio
|
||||
async def test_offloaded_counts_do_not_borrow_from_the_shared_thread_pool():
|
||||
encoder: Final = _SlowEncoder()
|
||||
count: Final = _get_exact_count_function(None, {"type": "huggingface_tokenizer", "tokenizer": encoder})
|
||||
count: Final = _resolve_exact_count(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}).count
|
||||
shared_pool: Final = anyio.to_thread.current_default_thread_limiter()
|
||||
burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
|
||||
|
||||
|
|
@ -279,7 +281,7 @@ def test_token_counter_applies_the_default_cap():
|
|||
max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS
|
||||
prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars]
|
||||
over_the_cap: Final = prose + "a" * 200_000
|
||||
exact: Final = _get_exact_count_function("gpt-5.6")(over_the_cap)
|
||||
exact: Final = _resolve_exact_count("gpt-5.6").count(over_the_cap)
|
||||
|
||||
estimate: Final = token_counter_new(model="gpt-5.6", text=over_the_cap)
|
||||
|
||||
|
|
@ -3534,8 +3536,8 @@ def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encodin
|
|||
|
||||
def test_empty_custom_tokenizer_uses_model_tokenizer() -> None:
|
||||
text_value: Final = "A tokenizer fallback should preserve the model encoding."
|
||||
custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value)
|
||||
model_count: Final = _get_exact_count_function("gpt-3.5-turbo", None)(text_value)
|
||||
custom_count: Final = _resolve_exact_count("gpt-3.5-turbo", {}).count(text_value)
|
||||
model_count: Final = _resolve_exact_count("gpt-3.5-turbo", None).count(text_value)
|
||||
|
||||
assert custom_count == model_count
|
||||
|
||||
|
|
@ -3870,3 +3872,83 @@ def test_openai_model_check_holds_while_openai_models_are_registered_concurrentl
|
|||
)
|
||||
def test_has_only_text_content_accepts_only_plain_text_chat_messages(messages: list[object], text_only: bool) -> None:
|
||||
assert has_only_text_content(messages) is text_only
|
||||
|
||||
|
||||
_MEMOIZED_TEXT: Final = "a long turn of the conversation, resent with every request. " * 40
|
||||
|
||||
|
||||
class _CallCountingCounter:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, text: str) -> int:
|
||||
self.calls += 1
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
class _CallCountingEncoder:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def encode_batch_fast(self, texts: list[str]) -> list[list[int]]:
|
||||
self.calls += 1
|
||||
return [[0] * len(text) for text in texts]
|
||||
|
||||
|
||||
def test_token_count_memo_counts_a_repeated_long_text_once():
|
||||
counter: Final = _CallCountingCounter()
|
||||
count: Final = TokenCountMemo().wrap(object(), counter)
|
||||
|
||||
assert count(_MEMOIZED_TEXT) == count(_MEMOIZED_TEXT) == len(_MEMOIZED_TEXT) // 4
|
||||
assert counter.calls == 1
|
||||
|
||||
|
||||
def test_token_count_memo_counts_text_below_the_threshold_every_time():
|
||||
counter: Final = _CallCountingCounter()
|
||||
count: Final = TokenCountMemo(min_chars=1024).wrap(object(), counter)
|
||||
short_text: Final = _MEMOIZED_TEXT[:1023]
|
||||
|
||||
assert count(short_text) == count(short_text)
|
||||
assert counter.calls == 2
|
||||
|
||||
|
||||
def test_token_count_memo_evicts_the_least_recently_used_count():
|
||||
counter: Final = _CallCountingCounter()
|
||||
count: Final = TokenCountMemo(max_entries=2).wrap(object(), counter)
|
||||
first, second, third = (f"{index} {_MEMOIZED_TEXT}" for index in range(3))
|
||||
|
||||
tuple(map(count, (first, second, first, third)))
|
||||
assert counter.calls == 3
|
||||
count(first)
|
||||
assert counter.calls == 3
|
||||
count(second)
|
||||
assert counter.calls == 4
|
||||
|
||||
|
||||
def test_token_count_memo_holds_no_reference_to_the_text():
|
||||
memo: Final = TokenCountMemo()
|
||||
text: Final = "".join(("fresh ", _MEMOIZED_TEXT))
|
||||
references_before: Final = sys.getrefcount(text)
|
||||
|
||||
memo.wrap(object(), len)(text)
|
||||
|
||||
assert sys.getrefcount(text) == references_before
|
||||
|
||||
|
||||
def test_memoized_counts_match_uncached_counts_for_each_model_sharing_the_memo():
|
||||
text: Final = "Ünïcödé prose, code() and 🙂 emoji in one long message, counted for several models. " * 60
|
||||
models: Final = ("gpt-4o", "gpt-3.5-turbo", "claude-sonnet-4-5")
|
||||
uncached: Final = tuple(_resolve_exact_count(model).count(text) for model in models)
|
||||
|
||||
assert len(set(uncached)) == len(models)
|
||||
assert tuple(token_counter_new(model=model, text=text) for model in models * 2) == uncached * 2
|
||||
|
||||
|
||||
def test_custom_tokenizer_counts_are_not_memoized():
|
||||
encoder: Final = _CallCountingEncoder()
|
||||
custom_tokenizer: Final = {"type": "huggingface_tokenizer", "tokenizer": encoder}
|
||||
|
||||
counts: Final = tuple(token_counter_new(custom_tokenizer=custom_tokenizer, text=_MEMOIZED_TEXT) for _ in range(2))
|
||||
|
||||
assert counts == (len(_MEMOIZED_TEXT), len(_MEMOIZED_TEXT))
|
||||
assert encoder.calls == 2
|
||||
|
|
|
|||
|
|
@ -2814,10 +2814,10 @@ def test_build_summary_messages_keeps_midturn_system_correction_in_place():
|
|||
|
||||
|
||||
async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -2828,7 +2828,7 @@ async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch):
|
|||
|
||||
monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "claude-haiku-4-5")
|
||||
warm_tokenizer(MODEL)
|
||||
messages = [{"role": "user", "content": text * 100}, *_simple_messages()]
|
||||
messages = [{"role": "user", "content": uncounted_text()}, *_simple_messages()]
|
||||
result, took, lags = await timed_with_loop_lags(
|
||||
lambda: apply_compact_20260112(
|
||||
model=MODEL,
|
||||
|
|
|
|||
|
|
@ -131,15 +131,15 @@ async def test_malformed_edit_entries_are_skipped():
|
|||
|
||||
|
||||
async def test_sync_editor_counts_tokens_off_the_event_loop():
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
warm_tokenizer(MODEL)
|
||||
messages = [{"role": "user", "content": text * 100}, *_history_with_two_tool_pairs()]
|
||||
messages = [{"role": "user", "content": uncounted_text()}, *_history_with_two_tool_pairs()]
|
||||
|
||||
result, took, lags = await timed_with_loop_lags(
|
||||
lambda: apply_context_management(
|
||||
|
|
|
|||
|
|
@ -3404,10 +3404,10 @@ async def test_post_call_success_hook_contains_header_merge_failures(rate_limite
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loop(rate_limiter):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -3416,7 +3416,7 @@ async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loo
|
|||
warm_tokenizer("claude-fable-5")
|
||||
data: dict[str, object] = {
|
||||
"model": "claude-fable-5",
|
||||
"messages": [{"role": "user", "content": text * 100}],
|
||||
"messages": [{"role": "user", "content": uncounted_text()}],
|
||||
}
|
||||
itpm_descriptor = {
|
||||
"key": PROJECT_ITPM_DESCRIPTOR_KEY,
|
||||
|
|
|
|||
|
|
@ -165,10 +165,10 @@ def _interrupted_anthropic_stream(model: str, output_text: str) -> list[bytes]:
|
|||
async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(caplog):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -190,7 +190,7 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event
|
|||
request_body={"model": model, "stream": True},
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
|
||||
raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()),
|
||||
end_time=datetime.now(),
|
||||
model=model,
|
||||
)
|
||||
|
|
@ -206,10 +206,10 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event
|
|||
async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(caplog):
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -232,7 +232,7 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(
|
|||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
request_body={"model": model, "stream": True},
|
||||
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
|
||||
raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()),
|
||||
exception=RuntimeError("upstream closed the stream"),
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15711,10 +15711,10 @@ def test_settings_store_exposes_dashboard_saved_mcp_client_allowlist_to_the_mcp_
|
|||
|
||||
|
||||
async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_count(monkeypatch):
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -15722,7 +15722,9 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun
|
|||
warm_tokenizer("claude-fable-5")
|
||||
|
||||
response, took, lags = await timed_with_loop_lags(
|
||||
lambda: proxy_server_module.count_request_tokens(TokenCountRequest(model="claude-fable-5", prompt=text * 100))
|
||||
lambda: proxy_server_module.count_request_tokens(
|
||||
TokenCountRequest(model="claude-fable-5", prompt=uncounted_text())
|
||||
)
|
||||
)
|
||||
|
||||
assert response.total_tokens > 0
|
||||
|
|
|
|||
|
|
@ -4056,10 +4056,10 @@ def test_a_dispatched_failure_lifts_the_fields_the_spend_log_needs():
|
|||
async def test_a_dispatched_failure_is_counted_off_the_event_loop():
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -4070,7 +4070,7 @@ async def test_a_dispatched_failure_is_counted_off_the_event_loop():
|
|||
"first_api_call_start_time": 1700000000.0,
|
||||
"call_type": "acompletion",
|
||||
"model": "claude-fable-5",
|
||||
"messages": [{"role": "user", "content": text * 100}],
|
||||
"messages": [{"role": "user", "content": uncounted_text()}],
|
||||
}
|
||||
),
|
||||
"metadata": {},
|
||||
|
|
|
|||
|
|
@ -766,17 +766,17 @@ async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop():
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
warm_tokenizer("anthropic/claude-fable-5")
|
||||
check = PromptCachingDeploymentCheck(cache=DualCache())
|
||||
deployments = _deployments("anthropic/claude-fable-5", "anthropic/claude-fable-5")
|
||||
messages = cast(list[AllMessageValues], [{"role": "user", "content": text * 100}])
|
||||
messages = cast(list[AllMessageValues], [{"role": "user", "content": uncounted_text()}])
|
||||
|
||||
result, took, lags = await timed_with_loop_lags(
|
||||
lambda: check.async_filter_deployments(
|
||||
|
|
@ -790,10 +790,10 @@ async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_success_event_counts_the_prompt_off_the_event_loop():
|
||||
from tests.large_text import text
|
||||
from tests.unit.litellm_core_utils.event_loop_lag import (
|
||||
assert_loop_stayed_free,
|
||||
timed_with_loop_lags,
|
||||
uncounted_text,
|
||||
warm_tokenizer,
|
||||
)
|
||||
|
||||
|
|
@ -802,7 +802,14 @@ async def test_async_log_success_event_counts_the_prompt_off_the_event_loop():
|
|||
check = PromptCachingDeploymentCheck(cache=cache)
|
||||
messages = cast(
|
||||
list[AllMessageValues],
|
||||
[{"role": "user", "content": [{"type": "text", "text": text * 100, "cache_control": {"type": "ephemeral"}}]}],
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": uncounted_text(), "cache_control": {"type": "ephemeral"}}
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
standard_logging_object = {
|
||||
"call_type": "acompletion",
|
||||
|
|
|
|||
|
|
@ -4699,10 +4699,11 @@ async def test_a_chat_pre_call_count_does_not_wait_behind_long_embedding_batches
|
|||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
batch: Final = ["word " * 2100] * 76
|
||||
batches: Final = tuple(
|
||||
asyncio.create_task(router.aembedding(model="embed", input=batch))
|
||||
for _ in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS)
|
||||
asyncio.create_task(
|
||||
router.aembedding(model="embed", input=[f"{batch} {text} " + "word " * 2100 for text in range(76)])
|
||||
)
|
||||
for batch in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS)
|
||||
)
|
||||
await asyncio.wait_for(_until_every_count_slot_is_taken(), timeout=30)
|
||||
chat: Final = asyncio.create_task(router.acompletion(model="chat", messages=[{"role": "user", "content": "hi"}]))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue