diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs index 244401e6696..97ddefdcd2f 100644 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/token_counter.rs @@ -19,10 +19,11 @@ use tokio::sync::Semaphore; use crate::errors::RustBridgeDeclined; /// Counts the input tokens of a raw request body off the Python event loop with -/// the GIL released. Python owns which requests get here and what to do with -/// the count. At most one encode per core runs at a time; the rest wait in the -/// async task, where a cancelled Python awaiter drops them before any blocking -/// work is scheduled. +/// the GIL released, or of one string synchronously on the calling thread. +/// Python owns which requests get here and what to do with the count. At most +/// one request encode per core runs at a time; the rest wait in the async task, +/// where a cancelled Python awaiter drops them before any blocking work is +/// scheduled. #[pyclass(frozen)] pub(crate) struct TokenCounter { inner: Arc, @@ -95,6 +96,10 @@ impl TokenCounter { } } + fn count_text(&self, py: Python<'_>, text: &str) -> PyResult { + release_gil(py, || self.inner.count_text(text)).map_err(token_count_error_to_pyerr) + } + fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { let counter = Arc::clone(&self.inner); let encode_slots = Arc::clone(&self.encode_slots); diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bf37b1be2e4..85e2d6c34ed 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -617,7 +617,8 @@ def _get_exact_count_function( ) -> TokenCounterFunction: """ Get the function to count tokens based on the model and custom tokenizer.""" - from litellm.utils import _select_tokenizer + from litellm.rust_bridge.token_counter import text_counter + from litellm.utils import _select_tokenizer, huggingface_tokenizer_kind if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) @@ -627,7 +628,12 @@ def _get_exact_count_function( def count_tokens(text: str) -> int: return len(tokenizer.encode_batch_fast([text])[0]) - return count_tokens + if model is None or huggingface_tokenizer_kind(model) != "anthropic": + return count_tokens + if tokenizer is not _select_tokenizer(model)["tokenizer"]: + return count_tokens + rust_count: Final = text_counter("anthropic") + return count_tokens if rust_count is None else _with_python_fallback(rust_count, count_tokens) elif tokenizer_json["type"] == "openai_tokenizer": encoding: Final = openai_tokenizer_encoding(model) @@ -645,6 +651,17 @@ def _get_exact_count_function( return _get_tiktoken_count_function(encode_length) +def _with_python_fallback(rust_count: TokenCounterFunction, python_count: TokenCounterFunction) -> TokenCounterFunction: + def count_tokens(text: str) -> int: + try: + return rust_count(text) + except RuntimeError as error: + verbose_logger.debug("Rust token counter failed, counting in Python: %s", error) + return python_count(text) + + return count_tokens + + def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" from litellm.utils import print_verbose diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..514174c7e90 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable from dataclasses import dataclass from functools import lru_cache from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables @@ -25,6 +25,9 @@ class RustTokenCounter(Protocol): def acount_request(self, body: bytes) -> Awaitable[object]: raise NotImplementedError + def count_text(self, text: str) -> int: + raise NotImplementedError + class RustTokenCounterFactory(Protocol): def __call__(self, tokenizer_json: str) -> RustTokenCounter: @@ -93,6 +96,22 @@ def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> Rust return factory.from_o200k_ranks(o200k_base_rank_file()) +def text_counter(tokenizer: RustTokenizer) -> Callable[[str], int] | None: + """The Rust per-string counter for `tokenizer`, `None` when the bridge is off or the extension is missing.""" + if not rust_enabled(): + return None + factory: Final = TOKEN_COUNTER.load() + if factory is None: + return None + try: + counter: Final = _counter(factory, tokenizer) + except (RuntimeError, ValueError) as error: + verbose_logger.debug("Rust token counter (%s) failed to load, counting in Python: %s", tokenizer, error) + return None + verbose_logger.debug("Rust token counter (%s) counting text", tokenizer) + return counter.count_text + + async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: if not rust_enabled(): return None diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index f19a8891609..0035ff42c34 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1602,3 +1602,177 @@ def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, def test_high_detail_image_token_upper_bound_is_reached_by_the_largest_high_res_image() -> None: assert calculate_img_tokens(_png_data_url(2000, 768), mode="high") == high_detail_image_token_upper_bound() assert calculate_img_tokens(_png_data_url(1, 1), mode="high") < high_detail_image_token_upper_bound() + + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +RUST_TEXTS: Final = ( + "Hello, how are you today?", + "I'VE got 1234567 things; it's \"fine\"...\r\n\r\n caf\u00e9 \u0645\u0631\u062d\u0628\u0627 \U0001f600 <|endoftext|>", + "x " * 5_000, + "", +) + + +class _FakeTextCounter: + def __init__(self, count: int | Exception) -> None: + self.count = count + self.texts: list[str] = [] + + def count_text(self, text: str) -> int: + self.texts.append(text) + if isinstance(self.count, Exception): + raise self.count + return self.count + + +class _FakeTextCounterFactory: + def __init__(self, count: int | Exception) -> None: + self.count = count + self.counters: list[_FakeTextCounter] = [] + + def __call__(self, tokenizer_json: str) -> _FakeTextCounter: + counter = _FakeTextCounter(self.count) + self.counters.append(counter) + return counter + + def from_cl100k_ranks(self, rank_file: str) -> _FakeTextCounter: + return self(rank_file) + + def from_o200k_ranks(self, rank_file: str) -> _FakeTextCounter: + return self(rank_file) + + +@pytest.fixture +def rust_bridge(monkeypatch: pytest.MonkeyPatch): + from litellm.rust_bridge import bindings, configuration + from litellm.rust_bridge import token_counter as bridge + + bridge.TOKEN_COUNTER.reset() + bridge._counter.cache_clear() + configuration.reset_rust_configuration() + monkeypatch.setattr(bindings, "get_native_bridge", lambda: object()) + yield bridge + bridge.TOKEN_COUNTER.reset() + bridge._counter.cache_clear() + configuration.reset_rust_configuration() + + +def test_anthropic_text_is_counted_by_rust_when_the_bridge_is_enabled(rust_bridge) -> None: + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(factory) + + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello") == 1_000 + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello again") == 1_000 + assert len(factory.counters) == 1 + assert factory.counters[0].texts == ["hello", "hello again"] + + +def test_anthropic_messages_are_counted_by_rust_when_the_bridge_is_enabled(rust_bridge) -> None: + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(factory) + + count: Final = token_counter_new(model=ANTHROPIC_MODEL, messages=[{"role": "user", "content": "hello"}]) + + assert count >= 2_000 + assert "hello" in factory.counters[0].texts + + +@pytest.mark.parametrize("model", ("gpt-4", "gpt-4o", "replicate/meta/llama-2-70b-chat")) +def test_only_the_anthropic_tokenizer_is_routed_to_rust(rust_bridge, model: str) -> None: + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(factory) + + assert token_counter_new(model=model, text="hello") < 1_000 + assert factory.counters == [] + + +def test_custom_huggingface_tokenizer_stays_in_python_for_anthropic_models(rust_bridge) -> None: + from tokenizers import Tokenizer + + from litellm.utils import claude_json_str + + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(factory) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": Tokenizer.from_str(claude_json_str)} + + assert token_counter_new(model=ANTHROPIC_MODEL, custom_tokenizer=custom, text="hello") < 1_000 + assert factory.counters == [] + + +def test_preselected_anthropic_tokenizer_is_counted_by_rust(rust_bridge) -> None: + """The proxy's token counting route selects the tokenizer itself and passes it in as `custom_tokenizer`.""" + from litellm.utils import _select_tokenizer + + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(factory) + + count: Final = token_counter_new( + model=ANTHROPIC_MODEL, custom_tokenizer=_select_tokenizer(ANTHROPIC_MODEL), text="hello" + ) + + assert count == 1_000 + assert [counter.texts for counter in factory.counters] == [["hello"]] + + +def test_disabled_bridge_counts_anthropic_text_in_python(rust_bridge) -> None: + factory: Final = _FakeTextCounterFactory(1_000) + litellm.rust(False) + rust_bridge.TOKEN_COUNTER.override(factory) + + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello") < 1_000 + assert factory.counters == [] + + +def test_missing_native_module_counts_anthropic_text_in_python(rust_bridge, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.rust_bridge import bindings + + litellm.rust(True) + monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) + litellm.rust(False) + python_count: Final = token_counter_new(model=ANTHROPIC_MODEL, text="hello") + litellm.rust(True) + + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello") == python_count + + +def test_rust_encode_failure_falls_back_to_python_per_string(rust_bridge) -> None: + litellm.rust(False) + python_count: Final = token_counter_new(model=ANTHROPIC_MODEL, text="hello") + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(_FakeTextCounterFactory(RuntimeError("encode failed"))) + + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello") == python_count + + +def test_rust_tokenizer_load_failure_falls_back_to_python(rust_bridge) -> None: + def failing_factory(tokenizer_json: str) -> _FakeTextCounter: + raise ValueError("tokenizer json rejected") + + litellm.rust(False) + python_count: Final = token_counter_new(model=ANTHROPIC_MODEL, text="hello") + litellm.rust(True) + rust_bridge.TOKEN_COUNTER.override(failing_factory) + + assert token_counter_new(model=ANTHROPIC_MODEL, text="hello") == python_count + + +@pytest.mark.parametrize("text", RUST_TEXTS) +def test_native_anthropic_text_count_matches_python(rust_bridge, monkeypatch: pytest.MonkeyPatch, text: str) -> None: + from litellm.rust_bridge import bindings + + native: Final = pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + messages: Final = [{"role": "system", "content": "You are terse."}, {"role": "user", "name": "bob", "content": text}] + + litellm.rust(False) + python_text: Final = token_counter_new(model=ANTHROPIC_MODEL, text=text) + python_messages: Final = token_counter_new(model=ANTHROPIC_MODEL, messages=messages) + litellm.rust(True) + + assert token_counter_new(model=ANTHROPIC_MODEL, text=text) == python_text + assert token_counter_new(model=ANTHROPIC_MODEL, messages=messages) == python_messages diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 0e0025f5194..f0ccef79be3 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -275,6 +275,9 @@ class _DecliningCounter: async def acount_request(self, body: bytes) -> object: raise _FakeDeclined("unsupported content block") + def count_text(self, text: str) -> int: + return len(text) + class _DecliningFactory: def __call__(self, tokenizer_json: str) -> _DecliningCounter: diff --git a/tests/test_litellm_rust/test_token_counter.py b/tests/test_litellm_rust/test_token_counter.py new file mode 100644 index 00000000000..dc3522a454f --- /dev/null +++ b/tests/test_litellm_rust/test_token_counter.py @@ -0,0 +1,41 @@ +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.token_counter import token_counter +from litellm.rust_bridge import bindings +from litellm.rust_bridge import token_counter as bridge + +pytestmark = pytest.mark.requires_rust_extension + +ANTHROPIC_MODEL: Final = "claude-sonnet-4-5-20250929" +TEXTS: Final = ( + "Hello, how are you today?", + "I'VE got 1234567 things; it's \"fine\"...\r\n\r\n café مرحبا 😀 <|endoftext|>", + "x " * 5_000, + "", +) + + +@pytest.fixture +def native_bridge() -> None: + bridge.TOKEN_COUNTER.reset() + bridge._counter.cache_clear() # pyright: ignore[reportPrivateUsage] # the counter cache is keyed on the factory object + assert bindings.get_native_bridge() is not None + + +@pytest.mark.parametrize("text", TEXTS) +def test_native_anthropic_text_count_matches_python(native_bridge: None, text: str) -> None: + messages: Final = [{"role": "system", "content": "You are terse."}, {"role": "user", "name": "bob", "content": text}] + + litellm.rust(False) + python_text: Final = token_counter(model=ANTHROPIC_MODEL, text=text) + python_messages: Final = token_counter(model=ANTHROPIC_MODEL, messages=messages) + litellm.rust(True) + + rust_count: Final = bridge.text_counter("anthropic") + assert rust_count is not None + assert rust_count(text) == python_text + assert token_counter(model=ANTHROPIC_MODEL, text=text) == python_text + assert token_counter(model=ANTHROPIC_MODEL, messages=messages) == python_messages