From d4e2f2fff79f35d162fe6e445bbe8d12e989d9ec Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 11 Sep 2026 23:51:38 +0000 Subject: [PATCH] feat(rust): count TPM admission tokens in Rust for plain chat requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../hooks/parallel_request_limiter_v3.py | 39 +++- litellm/rust_bridge/token_counter.py | 27 ++- .../hooks/test_parallel_request_limiter_v3.py | 196 ++++++++++++++++++ .../rust_bridge/test_token_counter.py | 106 ++++++++++ 4 files changed, 366 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index c398abff099..612b07be059 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -59,6 +59,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) +from litellm.rust_bridge.token_counter import count_chat_input_tokens from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( @@ -3283,6 +3284,42 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type) return estimated_input_tokens + audio_token_estimate + def _counts_messages_as_chat(self, data: Mapping[str, object], call_type: str | None) -> bool: + return ( + data.get("prompt") is None + and data.get("input") is None + and call_type not in RESPONSES_API_CALL_TYPES + and call_type not in RERANK_API_CALL_TYPES + and call_type not in TEXT_COMPLETION_API_CALL_TYPES + and call_type not in GOOGLE_GENAI_NATIVE_CALL_TYPES + and not self._is_embedding_request(data, call_type) + ) + + async def _estimate_admission_input_tokens( + self, data: Mapping[str, object], model: str | None, call_type: str | None + ) -> int: + """ + ``_estimate_precise_input_tokens`` with the Rust counter in front of it for plain chat bodies, the one + shape whose Rust count equals ``token_counter``'s. Rust declines image and audio blocks, so those and + every other API shape keep the Python path with its image defaults and audio add-on. + """ + messages: Final = data.get("messages") + rust_count: Final = ( + await count_chat_input_tokens( + model=model or "", + messages=messages, + tools=data.get("tools"), + tool_choice=data.get("tool_choice"), + ) + if isinstance(messages, list) and self._counts_messages_as_chat(data, call_type) + else None + ) + if rust_count is not None: + return rust_count + return await offload_token_count(self._estimate_precise_input_tokens)( + data=data, model=model, call_type=call_type + ) + async def _reserve_project_io_tokens_or_raise( self, descriptors: Sequence[RateLimitDescriptor], @@ -3328,7 +3365,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, ) - raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( + raw_estimated_input_tokens: Final = await self._estimate_admission_input_tokens( data=data, model=requested_model, call_type=call_type ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..d41b78421cb 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -2,11 +2,12 @@ from __future__ import annotations -from collections.abc import Awaitable +from collections.abc import Awaitable, Sequence from dataclasses import dataclass from functools import lru_cache from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables +import orjson from pydantic import TypeAdapter import litellm @@ -112,3 +113,27 @@ async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputToke return None verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) return attempt.value + + +async def count_chat_input_tokens( + model: str, + messages: Sequence[object], + tools: object = None, + tool_choice: object = None, +) -> int | None: + """Rust count of an already-parsed chat body, `None` when Python must count. + + Callers holding Python messages instead of the raw request bytes pay one `orjson.dumps` of the counted fields. + Pydantic messages and other non-JSON objects are left to Python, as are the blocks Rust declines.""" + if not rust_enabled() or TOKEN_COUNTER.load() is None: + return None + tokenizer: Final = rust_tokenizer(model) + if tokenizer is None: + return None + try: + body: Final = orjson.dumps({"model": model, "messages": messages, "tools": tools, "tool_choice": tool_choice}) + except orjson.JSONEncodeError as error: + verbose_logger.debug("Rust token counter (%s) skipped an unserializable body: %s", tokenizer, error) + return None + count: Final = await count_input_tokens(body, tokenizer) + return None if count is None else count.input_tokens diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 10c0bb88a82..0373b7e5924 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3,6 +3,8 @@ Unit Tests for the max parallel request limiter v3 for the proxy """ import asyncio +import base64 +import json import logging import os import sys @@ -6284,3 +6286,197 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_ assert isinstance(values, list) assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == [] assert any("circuit breaker is open" in record.getMessage() for record in caplog.records) + + +class _FakeRustDeclined(Exception): + pass + + +class _FakeRustUpstream(Exception): + pass + + +class _FakeRustNative: + RustBridgeDeclined = _FakeRustDeclined + RustUpstreamError = _FakeRustUpstream + + +RUST_CHAT_INPUT_TOKENS = 777 + + +class _RustChatCounter: + def __init__(self, bodies: list[bytes], declined: bool) -> None: + self.bodies = bodies + self.declined = declined + + async def acount_request(self, body: bytes) -> object: + self.bodies.append(body) + if self.declined: + raise _FakeRustDeclined("unsupported content block") + return {"model": "", "input_tokens": RUST_CHAT_INPUT_TOKENS} + + +class _RustChatFactory: + """Stands in for the native `TokenCounter` class and records every body handed to any tokenizer.""" + + def __init__(self, declined: bool = False) -> None: + self.bodies: list[bytes] = [] + self.declined = declined + + def __call__(self, tokenizer_json: str) -> _RustChatCounter: + return _RustChatCounter(self.bodies, self.declined) + + def from_cl100k_ranks(self, rank_file: str) -> _RustChatCounter: + return _RustChatCounter(self.bodies, self.declined) + + def from_o200k_ranks(self, rank_file: str) -> _RustChatCounter: + return _RustChatCounter(self.bodies, self.declined) + + +@pytest.fixture +def rust_chat_factory(monkeypatch: pytest.MonkeyPatch): + from litellm.rust_bridge import bindings, configuration + from litellm.rust_bridge import token_counter as rust_token_counter + + monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeRustNative()) + rust_token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + factory = _RustChatFactory() + litellm.rust(True) + rust_token_counter.TOKEN_COUNTER.override(factory) + yield factory + rust_token_counter.TOKEN_COUNTER.reset() + rust_token_counter._counter.cache_clear() + configuration.reset_rust_configuration() + + +CHAT_BODY_FOR_RUST = { + "model": "gpt-4o", + "messages": [{"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"}], + "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + "max_tokens": 50, + "metadata": {"user_api_key": "sk-hidden"}, +} + + +@pytest.mark.asyncio +async def test_project_itpm_reservation_counts_plain_chat_bodies_in_rust(rust_chat_factory: _RustChatFactory): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-rust-itpm"), + project_id="proj-rust", + project_metadata={"model_itpm_limit": {"gpt-4o": 10_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data={**CHAT_BODY_FOR_RUST, "litellm_call_id": "rust-project-call"}, + call_type="completion", + ) + + stash = get_request_stash() + assert stash is not None and stash.itpm_reserved_tokens == RUST_CHAT_INPUT_TOKENS + assert [json.loads(body) for body in rust_chat_factory.bodies] == [ + { + "model": "gpt-4o", + "messages": CHAT_BODY_FOR_RUST["messages"], + "tools": CHAT_BODY_FOR_RUST["tools"], + "tool_choice": "auto", + } + ] + + +@pytest.mark.asyncio +async def test_admission_input_tokens_fall_back_to_python_when_rust_declines(rust_chat_factory: _RustChatFactory): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + rust_chat_factory.declined = True + audio_body = { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "transcribe"}, + { + "type": "input_audio", + "input_audio": {"data": base64.b64encode(b"a" * 4_000).decode(), "format": "wav"}, + }, + ], + } + ], + } + python_estimate = handler._estimate_precise_input_tokens(audio_body, model="gpt-4o", call_type="acompletion") + + estimate = await handler._estimate_admission_input_tokens(audio_body, model="gpt-4o", call_type="acompletion") + + assert len(rust_chat_factory.bodies) == 1 + assert estimate == python_estimate + assert estimate > handler._estimate_precise_input_tokens( + {"model": "gpt-4o", "messages": [{"role": "user", "content": "transcribe"}]}, + model="gpt-4o", + call_type="acompletion", + ) + + +@pytest.mark.asyncio +async def test_admission_input_tokens_stay_in_python_when_rust_is_disabled(rust_chat_factory: _RustChatFactory): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + litellm.rust(False) + + estimate = await handler._estimate_admission_input_tokens( + CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion" + ) + + assert rust_chat_factory.bodies == [] + assert estimate == handler._estimate_precise_input_tokens( + CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("data", "call_type"), + ( + ({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, "aresponses"), + ({"model": "gpt-4o", "input": ["hello"], "messages": [{"role": "user", "content": "ignored"}]}, "aembedding"), + ( + {"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "ignored"}]}, + "atext_completion", + ), + ( + {"model": "gpt-4o", "query": "q", "documents": ["d"], "messages": [{"role": "user", "content": "x"}]}, + "arerank", + ), + ({"model": "gpt-4o", "contents": [{"role": "user", "parts": [{"text": "hello"}]}]}, "agenerate_content"), + ({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, None), + ({"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "both"}]}, "acompletion"), + ({"model": "gpt-4o", "messages": "not a list"}, "acompletion"), + ), +) +async def test_admission_input_tokens_keep_non_chat_shapes_in_python( + rust_chat_factory: _RustChatFactory, data: dict, call_type: str | None +): + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + + estimate = await handler._estimate_admission_input_tokens(data, model="gpt-4o", call_type=call_type) + + assert rust_chat_factory.bodies == [] + assert estimate == handler._estimate_precise_input_tokens(data, model="gpt-4o", call_type=call_type) + + +@pytest.mark.asyncio +async def test_admission_input_tokens_keep_models_without_a_rust_tokenizer_in_python( + rust_chat_factory: _RustChatFactory, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr( + litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"} + ) + handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache())) + data = {**CHAT_BODY_FOR_RUST, "model": "text-davinci-003"} + + estimate = await handler._estimate_admission_input_tokens(data, model="text-davinci-003", call_type="acompletion") + + assert rust_chat_factory.bodies == [] + assert estimate == handler._estimate_precise_input_tokens(data, model="text-davinci-003", call_type="acompletion") diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 71aa79cc4bb..afc45cec319 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -304,6 +304,70 @@ def test_disabled_token_counter_declines_every_model(monkeypatch: pytest.MonkeyP assert bridge.rust_tokenizer(O200K_MODEL) is None +CHAT_MESSAGES: Final = ({"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"}) +CHAT_TOOLS: Final = ({"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL)) +async def test_chat_count_serializes_only_the_counted_fields(model: str) -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + bridge.TOKEN_COUNTER.override(factory) + + count: Final = await bridge.count_chat_input_tokens( + model=model, messages=CHAT_MESSAGES, tools=CHAT_TOOLS, tool_choice="auto" + ) + + assert count == 42 + assert [json.loads(body) for counter in factory.counters for body in counter.bodies] == [ + {"model": model, "messages": list(CHAT_MESSAGES), "tools": list(CHAT_TOOLS), "tool_choice": "auto"} + ] + + +@pytest.mark.asyncio +async def test_chat_count_declines_before_serializing_when_the_bridge_is_off() -> None: + factory: Final = _RecordingFactory() + litellm.rust(False) + bridge.TOKEN_COUNTER.override(factory) + + assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None + assert factory.counters == [] + + +@pytest.mark.asyncio +async def test_chat_count_declines_models_without_a_rust_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"} + ) + factory: Final = _RecordingFactory() + litellm.rust(True) + bridge.TOKEN_COUNTER.override(factory) + + assert await bridge.count_chat_input_tokens(model="text-davinci-003", messages=CHAT_MESSAGES) is None + assert factory.counters == [] + + +@pytest.mark.asyncio +async def test_chat_count_declines_when_the_native_counter_declines() -> None: + litellm.rust(True) + bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("unsupported content block"))) + + assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None + + +@pytest.mark.asyncio +async def test_chat_count_declines_an_unserializable_body() -> None: + factory: Final = _RecordingFactory() + litellm.rust(True) + bridge.TOKEN_COUNTER.override(factory) + + messages: Final = ({"role": "user", "content": object()},) + + assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None + assert factory.counters == [] + + PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = ( {"model": MODEL, "messages": [{"role": "user", "content": "Hello, how are you today?"}]}, { @@ -442,3 +506,45 @@ async def test_native_declines_shapes_python_prices_differently( litellm.rust(True) assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL)) +@pytest.mark.parametrize( + "request_body", tuple(body for body in PARITY_REQUESTS if isinstance(body.get("messages"), list)) +) +async def test_native_chat_count_matches_python_token_counter( + monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str +) -> None: + native: Final = pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + litellm.rust(True) + messages: Final = request_body["messages"] + assert isinstance(messages, list) + + rust_count: Final = await bridge.count_chat_input_tokens( + model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice") + ) + + assert rust_count == litellm.token_counter( + model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice") + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "block", + ( + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}}, + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + ), +) +async def test_native_chat_count_declines_multimodal_blocks( + monkeypatch: pytest.MonkeyPatch, block: dict[str, object] +) -> None: + native: Final = pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) + litellm.rust(True) + messages: Final = ({"role": "user", "content": [{"type": "text", "text": "look"}, block]},) + + assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None