perf(token_counter): memoize token counts of long texts per tokenizer (#51904)

* perf(token_counter): memoize token counts of long texts per tokenizer

* test(prompt_caching): count a prompt no other test has counted in the loop lag checks

* fix(token_counter): key the count memo on the string hash instead of blake2b

* test(router): embed distinct texts so the batches are counted in full

* test: time token counts over text no earlier count has memoized
This commit is contained in:
Nate Armstrong 2026-11-16 11:21:55 -08:00 • committed by GitHub
parent e4759def40
commit c60a320b05
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 202 additions and 51 deletions

View file

@ -5,7 +5,9 @@ import io
import math
import re
import struct
import threading
from bisect import bisect_right
from collections import OrderedDict
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from functools import cached_property
@ -1233,19 +1235,67 @@ def _evenly_spaced_samples(text: str, total_chars: int) -> tuple[str, ...]:
)
_MemoKey: TypeAlias = tuple[int, int, int]
_MemoEntry: TypeAlias = tuple[object, int]
class TokenCountMemo:
"""Least recently used token counts of long texts, so a conversation resent with one more turn only tokenizes
the new turn. Entries are keyed by the tokenizer's id and the text's length and hash, never the text itself, and
hold their tokenizer so that id cannot be reused by another one while the entry lives."""
__slots__ = ("_counts", "_lock", "_max_entries", "_min_chars")
def __init__(self, max_entries: int = 16_384, min_chars: int = 1024) -> None:
self._counts: Final[OrderedDict[_MemoKey, _MemoEntry]] = OrderedDict() # mutable-ok: an LRU evicts in place
self._lock: Final = threading.Lock()
self._max_entries: Final = max_entries
self._min_chars: Final = min_chars
def wrap(self, tokenizer: object, count: TokenCounterFunction) -> TokenCounterFunction:
def count_tokens(text: str) -> int:
if len(text) < self._min_chars:
return count(text)
key: Final = (id(tokenizer), len(text), hash(text))
with self._lock:
cached: Final = self._counts.get(key)
if cached is not None:
self._counts.move_to_end(key)
return cached[1]
counted: Final = count(text)
with self._lock:
self._counts[key] = (tokenizer, counted)
self._counts.move_to_end(key)
if len(self._counts) > self._max_entries:
self._counts.popitem(last=False)
return counted
return count_tokens
_TOKEN_COUNT_MEMO: Final = TokenCountMemo()
def _get_count_function(
model: str | None,
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
) -> TokenCounterFunction:
return _get_extrapolating_count_function(_get_exact_count_function(model, custom_tokenizer))
exact: Final = _resolve_exact_count(model, custom_tokenizer)
count: Final = _get_extrapolating_count_function(exact.count)
return count if custom_tokenizer is not None else _TOKEN_COUNT_MEMO.wrap(exact.tokenizer, count)
def _get_exact_count_function(
@dataclass(frozen=True, slots=True)
class _ExactCount:
tokenizer: "Encoding | HuggingFace"
count: TokenCounterFunction
def _resolve_exact_count(
model: str | None,
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
) -> TokenCounterFunction:
"""
Get the function to count tokens based on the model and custom tokenizer."""
) -> _ExactCount:
"""The tokenizer and the exact count function for the model and custom tokenizer."""
from litellm.utils import select_tokenizer
if model is not None or custom_tokenizer is not None:
@ -1256,7 +1306,7 @@ def _get_exact_count_function(
def encode_length(text: str) -> int:
return _encoding_count(default_encoding, text)
return _get_tiktoken_count_function(encode_length)
return _ExactCount(default_encoding, _get_tiktoken_count_function(encode_length))
if tokenizer_json["type"] == "huggingface_tokenizer":
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
@ -1265,14 +1315,14 @@ def _get_exact_count_function(
return tokenizer.count(text)
return len(tokenizer.encode_batch_fast([text])[0])
return count_tokens
return _ExactCount(tokenizer, count_tokens)
if tokenizer_json["type"] == "openai_tokenizer":
encoding: Final = openai_tokenizer_encoding(model or "")
def encode_length(text: str) -> int:
return _encoding_count(encoding, text)
return _get_tiktoken_count_function(encode_length)
return _ExactCount(encoding, _get_tiktoken_count_function(encode_length))
raise ValueError("Unsupported tokenizer type")

View file

@ -100,10 +100,10 @@ class _AgentChunk:
@pytest.mark.asyncio
async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -121,12 +121,12 @@ async def test_stream_completion_counts_tokens_off_the_event_loop(monkeypatch):
)
async def _stream():
yield _AgentChunk(text * 100)
yield _AgentChunk(uncounted_text())
iterator = A2AStreamingIterator(
stream=_stream(),
request=SimpleNamespace(
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": text * 100}]})
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]})
),
logging_obj=logging_obj,
agent_name="test-agent",

View file

@ -729,10 +729,10 @@ async def test_streamed_send_is_timed_from_before_the_agent_card_fetch(
@pytest.mark.asyncio
async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -745,11 +745,11 @@ async def test_asend_message_counts_usage_off_the_event_loop(monkeypatch):
reply = _conv.pb2_v10.StreamResponse()
reply.message.message_id = "reply-1"
reply.message.role = _conv.pb2_v10.Role.ROLE_AGENT
reply.message.parts.add().text = text * 100
reply.message.parts.add().text = uncounted_text()
request = SendMessageRequest(
id="r1",
params=MessageSendParams(
message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": text * 100}]}
message={"messageId": "m1", "role": "user", "parts": [{"kind": "text", "text": uncounted_text()}]}
),
)

View file

@ -995,10 +995,10 @@ def test_qdrant_semantic_cache_defaults_embedding_timeout():
@pytest.mark.asyncio
async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -1019,7 +1019,7 @@ async def test_qdrant_async_embedding_truncates_off_the_event_loop(monkeypatch):
_router_proxy_module(router, "sem-embed"),
)
response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
response, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text()))
assert response["data"][0]["embedding"] == [0.1, 0.2]
assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5

View file

@ -1430,10 +1430,10 @@ def test_redis_semantic_cache_defaults_embedding_timeout():
@pytest.mark.asyncio
async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -1450,7 +1450,7 @@ async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch):
router.aembedding = AsyncMock(return_value={"data": [{"embedding": [0.1, 0.2]}]})
_proxy_with_router(monkeypatch, router, "sem-embed")
embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(text * 100))
embedding, took, lags = await timed_with_loop_lags(lambda: cache._get_async_embedding(uncounted_text()))
assert embedding == [0.1, 0.2]
assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5

View file

@ -527,17 +527,17 @@ async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
@pytest.mark.asyncio
async def test_pre_call_hook_counts_tokens_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
model = "anthropic/claude-fable-5"
warm_tokenizer(model)
logger = CompressionInterceptionLogger(compression_trigger=10_000_000)
messages = [{"role": "user", "content": text * 100}]
messages = [{"role": "user", "content": uncounted_text()}]
kwargs = {"model": model, "messages": messages}
result, took, lags = await timed_with_loop_lags(

View file

@ -1,14 +1,22 @@
import asyncio
import gc
import time
import uuid
from collections.abc import Awaitable, Callable
from typing import Final, TypeVar
import litellm
from tests.large_text import text
T = TypeVar("T")
def uncounted_text() -> str:
"""`tests.large_text.text` 100 times behind a fresh prefix, so no earlier count can have cached its token count and
a timed count of it is a full tokenization."""
return f"{uuid.uuid4()} {text * 100}"
def warm_tokenizer(model: str) -> None:
litellm.token_counter(model=model, text="load the tokenizer before anything is timed")

View file

@ -9497,17 +9497,18 @@ class TestContinuationDisqualifiers:
@pytest.mark.asyncio
async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
model = "gpt-5.6-luna"
warm_tokenizer(model)
messages = [{"role": "user", "content": text * 100}]
content_chunks = [_make_chunk(text) for _ in range(100)]
messages = [{"role": "user", "content": uncounted_text()}]
completion = uncounted_text()
content_chunks = [_make_chunk(completion[start : start + 25_000]) for start in range(0, len(completion), 25_000)]
stop_chunk = ModelResponseStream(
id="test",
created=1741037890,
@ -9536,7 +9537,7 @@ async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
chunks, took, lags = await timed_with_loop_lags(consume)
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == text * 100
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == completion
assert chunks[-1].usage.prompt_tokens > 100_000
assert chunks[-1].usage.completion_tokens > 100_000
assert_loop_stayed_free(took, lags)

View file

@ -35,12 +35,13 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.token_counter import (
TokenCountMemo,
UnsupportedContentTypeError,
_count_content_list,
_encoding_count,
_get_exact_count_function,
_get_extrapolating_count_function,
_get_tiktoken_count_function,
_resolve_exact_count,
calculate_img_tokens,
estimate_image_tokens,
has_only_text_content,
@ -57,6 +58,7 @@ from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
from tests.unit.litellm_core_utils.messages_with_counts import (
@ -175,7 +177,7 @@ async def test_huggingface_count_in_a_worker_thread_leaves_the_event_loop_free()
warm_tokenizer("claude-fable-5")
tokens, took, lags = await timed_with_loop_lags(
lambda: asyncify(token_counter_new)(model="claude-fable-5", text=text * 100)
lambda: asyncify(token_counter_new)(model="claude-fable-5", text=uncounted_text())
)
assert tokens > 0
@ -220,7 +222,7 @@ class _SlowEncoder:
@pytest.mark.asyncio
async def test_offloaded_counts_do_not_borrow_from_the_shared_thread_pool():
encoder: Final = _SlowEncoder()
count: Final = _get_exact_count_function(None, {"type": "huggingface_tokenizer", "tokenizer": encoder})
count: Final = _resolve_exact_count(None, {"type": "huggingface_tokenizer", "tokenizer": encoder}).count
shared_pool: Final = anyio.to_thread.current_default_thread_limiter()
burst: Final = 2 * TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
@ -279,7 +281,7 @@ def test_token_counter_applies_the_default_cap():
max_exact_chars: Final = litellm.constants.TOKEN_COUNTER_MAX_EXACT_CHARS
prose: Final = ("The quick brown fox jumps over the lazy dog. " * (max_exact_chars // 45 + 1))[:max_exact_chars]
over_the_cap: Final = prose + "a" * 200_000
exact: Final = _get_exact_count_function("gpt-5.6")(over_the_cap)
exact: Final = _resolve_exact_count("gpt-5.6").count(over_the_cap)
estimate: Final = token_counter_new(model="gpt-5.6", text=over_the_cap)
@ -3534,8 +3536,8 @@ def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encodin
def test_empty_custom_tokenizer_uses_model_tokenizer() -> None:
text_value: Final = "A tokenizer fallback should preserve the model encoding."
custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value)
model_count: Final = _get_exact_count_function("gpt-3.5-turbo", None)(text_value)
custom_count: Final = _resolve_exact_count("gpt-3.5-turbo", {}).count(text_value)
model_count: Final = _resolve_exact_count("gpt-3.5-turbo", None).count(text_value)
assert custom_count == model_count
@ -3870,3 +3872,83 @@ def test_openai_model_check_holds_while_openai_models_are_registered_concurrentl
)
def test_has_only_text_content_accepts_only_plain_text_chat_messages(messages: list[object], text_only: bool) -> None:
assert has_only_text_content(messages) is text_only
_MEMOIZED_TEXT: Final = "a long turn of the conversation, resent with every request. " * 40
class _CallCountingCounter:
def __init__(self) -> None:
self.calls = 0
def __call__(self, text: str) -> int:
self.calls += 1
return len(text) // 4
class _CallCountingEncoder:
def __init__(self) -> None:
self.calls = 0
def encode_batch_fast(self, texts: list[str]) -> list[list[int]]:
self.calls += 1
return [[0] * len(text) for text in texts]
def test_token_count_memo_counts_a_repeated_long_text_once():
counter: Final = _CallCountingCounter()
count: Final = TokenCountMemo().wrap(object(), counter)
assert count(_MEMOIZED_TEXT) == count(_MEMOIZED_TEXT) == len(_MEMOIZED_TEXT) // 4
assert counter.calls == 1
def test_token_count_memo_counts_text_below_the_threshold_every_time():
counter: Final = _CallCountingCounter()
count: Final = TokenCountMemo(min_chars=1024).wrap(object(), counter)
short_text: Final = _MEMOIZED_TEXT[:1023]
assert count(short_text) == count(short_text)
assert counter.calls == 2
def test_token_count_memo_evicts_the_least_recently_used_count():
counter: Final = _CallCountingCounter()
count: Final = TokenCountMemo(max_entries=2).wrap(object(), counter)
first, second, third = (f"{index} {_MEMOIZED_TEXT}" for index in range(3))
tuple(map(count, (first, second, first, third)))
assert counter.calls == 3
count(first)
assert counter.calls == 3
count(second)
assert counter.calls == 4
def test_token_count_memo_holds_no_reference_to_the_text():
memo: Final = TokenCountMemo()
text: Final = "".join(("fresh ", _MEMOIZED_TEXT))
references_before: Final = sys.getrefcount(text)
memo.wrap(object(), len)(text)
assert sys.getrefcount(text) == references_before
def test_memoized_counts_match_uncached_counts_for_each_model_sharing_the_memo():
text: Final = "Ünïcödé prose, code() and 🙂 emoji in one long message, counted for several models. " * 60
models: Final = ("gpt-4o", "gpt-3.5-turbo", "claude-sonnet-4-5")
uncached: Final = tuple(_resolve_exact_count(model).count(text) for model in models)
assert len(set(uncached)) == len(models)
assert tuple(token_counter_new(model=model, text=text) for model in models * 2) == uncached * 2
def test_custom_tokenizer_counts_are_not_memoized():
encoder: Final = _CallCountingEncoder()
custom_tokenizer: Final = {"type": "huggingface_tokenizer", "tokenizer": encoder}
counts: Final = tuple(token_counter_new(custom_tokenizer=custom_tokenizer, text=_MEMOIZED_TEXT) for _ in range(2))
assert counts == (len(_MEMOIZED_TEXT), len(_MEMOIZED_TEXT))
assert encoder.calls == 2

View file

@ -2814,10 +2814,10 @@ def test_build_summary_messages_keeps_midturn_system_correction_in_place():
async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -2828,7 +2828,7 @@ async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch):
monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "claude-haiku-4-5")
warm_tokenizer(MODEL)
messages = [{"role": "user", "content": text * 100}, *_simple_messages()]
messages = [{"role": "user", "content": uncounted_text()}, *_simple_messages()]
result, took, lags = await timed_with_loop_lags(
lambda: apply_compact_20260112(
model=MODEL,

View file

@ -131,15 +131,15 @@ async def test_malformed_edit_entries_are_skipped():
async def test_sync_editor_counts_tokens_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
warm_tokenizer(MODEL)
messages = [{"role": "user", "content": text * 100}, *_history_with_two_tool_pairs()]
messages = [{"role": "user", "content": uncounted_text()}, *_history_with_two_tool_pairs()]
result, took, lags = await timed_with_loop_lags(
lambda: apply_context_management(

View file

@ -3404,10 +3404,10 @@ async def test_post_call_success_hook_contains_header_merge_failures(rate_limite
@pytest.mark.asyncio
async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loop(rate_limiter):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -3416,7 +3416,7 @@ async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loo
warm_tokenizer("claude-fable-5")
data: dict[str, object] = {
"model": "claude-fable-5",
"messages": [{"role": "user", "content": text * 100}],
"messages": [{"role": "user", "content": uncounted_text()}],
}
itpm_descriptor = {
"key": PROJECT_ITPM_DESCRIPTOR_KEY,

View file

@ -165,10 +165,10 @@ def _interrupted_anthropic_stream(model: str, output_text: str) -> list[bytes]:
async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event_loop(caplog):
from unittest.mock import AsyncMock
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -190,7 +190,7 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event
request_body={"model": model, "stream": True},
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()),
end_time=datetime.now(),
model=model,
)
@ -206,10 +206,10 @@ async def test_interrupted_anthropic_stream_recovers_output_tokens_off_the_event
async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(caplog):
from unittest.mock import AsyncMock
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -232,7 +232,7 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop(
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
request_body={"model": model, "stream": True},
raw_bytes=_interrupted_anthropic_stream(model, text * 100),
raw_bytes=_interrupted_anthropic_stream(model, uncounted_text()),
exception=RuntimeError("upstream closed the stream"),
)
)

View file

@ -15711,10 +15711,10 @@ def test_settings_store_exposes_dashboard_saved_mcp_client_allowlist_to_the_mcp_
async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_count(monkeypatch):
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -15722,7 +15722,9 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun
warm_tokenizer("claude-fable-5")
response, took, lags = await timed_with_loop_lags(
lambda: proxy_server_module.count_request_tokens(TokenCountRequest(model="claude-fable-5", prompt=text * 100))
lambda: proxy_server_module.count_request_tokens(
TokenCountRequest(model="claude-fable-5", prompt=uncounted_text())
)
)
assert response.total_tokens > 0

View file

@ -4056,10 +4056,10 @@ def test_a_dispatched_failure_lifts_the_fields_the_spend_log_needs():
async def test_a_dispatched_failure_is_counted_off_the_event_loop():
from unittest.mock import AsyncMock, patch
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -4070,7 +4070,7 @@ async def test_a_dispatched_failure_is_counted_off_the_event_loop():
"first_api_call_start_time": 1700000000.0,
"call_type": "acompletion",
"model": "claude-fable-5",
"messages": [{"role": "user", "content": text * 100}],
"messages": [{"role": "user", "content": uncounted_text()}],
}
),
"metadata": {},

View file

@ -766,17 +766,17 @@ async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost
@pytest.mark.asyncio
async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
warm_tokenizer("anthropic/claude-fable-5")
check = PromptCachingDeploymentCheck(cache=DualCache())
deployments = _deployments("anthropic/claude-fable-5", "anthropic/claude-fable-5")
messages = cast(list[AllMessageValues], [{"role": "user", "content": text * 100}])
messages = cast(list[AllMessageValues], [{"role": "user", "content": uncounted_text()}])
result, took, lags = await timed_with_loop_lags(
lambda: check.async_filter_deployments(
@ -790,10 +790,10 @@ async def test_async_filter_deployments_counts_the_prompt_off_the_event_loop():
@pytest.mark.asyncio
async def test_async_log_success_event_counts_the_prompt_off_the_event_loop():
from tests.large_text import text
from tests.unit.litellm_core_utils.event_loop_lag import (
assert_loop_stayed_free,
timed_with_loop_lags,
uncounted_text,
warm_tokenizer,
)
@ -802,7 +802,14 @@ async def test_async_log_success_event_counts_the_prompt_off_the_event_loop():
check = PromptCachingDeploymentCheck(cache=cache)
messages = cast(
list[AllMessageValues],
[{"role": "user", "content": [{"type": "text", "text": text * 100, "cache_control": {"type": "ephemeral"}}]}],
[
{
"role": "user",
"content": [
{"type": "text", "text": uncounted_text(), "cache_control": {"type": "ephemeral"}}
],
}
],
)
standard_logging_object = {
"call_type": "acompletion",

View file

@ -4699,10 +4699,11 @@ async def test_a_chat_pre_call_count_does_not_wait_behind_long_embedding_batches
],
enable_pre_call_checks=True,
)
batch: Final = ["word " * 2100] * 76
batches: Final = tuple(
asyncio.create_task(router.aembedding(model="embed", input=batch))
for _ in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS)
asyncio.create_task(
router.aembedding(model="embed", input=[f"{batch} {text} " + "word " * 2100 for text in range(76)])
)
for batch in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS)
)
await asyncio.wait_for(_until_every_count_slot_is_taken(), timeout=30)
chat: Final = asyncio.create_task(router.acompletion(model="chat", messages=[{"role": "user", "content": "hi"}]))