feat(rust): count TPM admission tokens in Rust for plain chat requests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-11 23:51:38 +00:00
parent e4706fa409
commit d4e2f2fff7
4 changed files with 366 additions and 2 deletions

View file

@ -59,6 +59,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.rust_bridge.token_counter import count_chat_input_tokens
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
from litellm.types.utils import (
@ -3283,6 +3284,42 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
return estimated_input_tokens + audio_token_estimate
def _counts_messages_as_chat(self, data: Mapping[str, object], call_type: str | None) -> bool:
return (
data.get("prompt") is None
and data.get("input") is None
and call_type not in RESPONSES_API_CALL_TYPES
and call_type not in RERANK_API_CALL_TYPES
and call_type not in TEXT_COMPLETION_API_CALL_TYPES
and call_type not in GOOGLE_GENAI_NATIVE_CALL_TYPES
and not self._is_embedding_request(data, call_type)
)
async def _estimate_admission_input_tokens(
self, data: Mapping[str, object], model: str | None, call_type: str | None
) -> int:
"""
``_estimate_precise_input_tokens`` with the Rust counter in front of it for plain chat bodies, the one
shape whose Rust count equals ``token_counter``'s. Rust declines image and audio blocks, so those and
every other API shape keep the Python path with its image defaults and audio add-on.
"""
messages: Final = data.get("messages")
rust_count: Final = (
await count_chat_input_tokens(
model=model or "",
messages=messages,
tools=data.get("tools"),
tool_choice=data.get("tool_choice"),
)
if isinstance(messages, list) and self._counts_messages_as_chat(data, call_type)
else None
)
if rust_count is not None:
return rust_count
return await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=model, call_type=call_type
)
async def _reserve_project_io_tokens_or_raise(
self,
descriptors: Sequence[RateLimitDescriptor],
@ -3328,7 +3365,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_otpm_limit,
call_type=call_type,
)
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
raw_estimated_input_tokens: Final = await self._estimate_admission_input_tokens(
data=data, model=requested_model, call_type=call_type
)
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)

View file

@ -2,11 +2,12 @@
from __future__ import annotations
from collections.abc import Awaitable
from collections.abc import Awaitable, Sequence
from dataclasses import dataclass
from functools import lru_cache
from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables
import orjson
from pydantic import TypeAdapter
import litellm
@ -112,3 +113,27 @@ async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputToke
return None
verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens)
return attempt.value
async def count_chat_input_tokens(
model: str,
messages: Sequence[object],
tools: object = None,
tool_choice: object = None,
) -> int | None:
"""Rust count of an already-parsed chat body, `None` when Python must count.
Callers holding Python messages instead of the raw request bytes pay one `orjson.dumps` of the counted fields.
Pydantic messages and other non-JSON objects are left to Python, as are the blocks Rust declines."""
if not rust_enabled() or TOKEN_COUNTER.load() is None:
return None
tokenizer: Final = rust_tokenizer(model)
if tokenizer is None:
return None
try:
body: Final = orjson.dumps({"model": model, "messages": messages, "tools": tools, "tool_choice": tool_choice})
except orjson.JSONEncodeError as error:
verbose_logger.debug("Rust token counter (%s) skipped an unserializable body: %s", tokenizer, error)
return None
count: Final = await count_input_tokens(body, tokenizer)
return None if count is None else count.input_tokens

View file

@ -3,6 +3,8 @@ Unit Tests for the max parallel request limiter v3 for the proxy
"""
import asyncio
import base64
import json
import logging
import os
import sys
@ -6284,3 +6286,197 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_
assert isinstance(values, list)
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
class _FakeRustDeclined(Exception):
pass
class _FakeRustUpstream(Exception):
pass
class _FakeRustNative:
RustBridgeDeclined = _FakeRustDeclined
RustUpstreamError = _FakeRustUpstream
RUST_CHAT_INPUT_TOKENS = 777
class _RustChatCounter:
def __init__(self, bodies: list[bytes], declined: bool) -> None:
self.bodies = bodies
self.declined = declined
async def acount_request(self, body: bytes) -> object:
self.bodies.append(body)
if self.declined:
raise _FakeRustDeclined("unsupported content block")
return {"model": "", "input_tokens": RUST_CHAT_INPUT_TOKENS}
class _RustChatFactory:
"""Stands in for the native `TokenCounter` class and records every body handed to any tokenizer."""
def __init__(self, declined: bool = False) -> None:
self.bodies: list[bytes] = []
self.declined = declined
def __call__(self, tokenizer_json: str) -> _RustChatCounter:
return _RustChatCounter(self.bodies, self.declined)
def from_cl100k_ranks(self, rank_file: str) -> _RustChatCounter:
return _RustChatCounter(self.bodies, self.declined)
def from_o200k_ranks(self, rank_file: str) -> _RustChatCounter:
return _RustChatCounter(self.bodies, self.declined)
@pytest.fixture
def rust_chat_factory(monkeypatch: pytest.MonkeyPatch):
from litellm.rust_bridge import bindings, configuration
from litellm.rust_bridge import token_counter as rust_token_counter
monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeRustNative())
rust_token_counter._counter.cache_clear()
configuration.reset_rust_configuration()
factory = _RustChatFactory()
litellm.rust(True)
rust_token_counter.TOKEN_COUNTER.override(factory)
yield factory
rust_token_counter.TOKEN_COUNTER.reset()
rust_token_counter._counter.cache_clear()
configuration.reset_rust_configuration()
CHAT_BODY_FOR_RUST = {
"model": "gpt-4o",
"messages": [{"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"}],
"tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}],
"tool_choice": "auto",
"max_tokens": 50,
"metadata": {"user_api_key": "sk-hidden"},
}
@pytest.mark.asyncio
async def test_project_itpm_reservation_counts_plain_chat_bodies_in_rust(rust_chat_factory: _RustChatFactory):
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
user_api_key_dict = UserAPIKeyAuth(
api_key=hash_token("sk-rust-itpm"),
project_id="proj-rust",
project_metadata={"model_itpm_limit": {"gpt-4o": 10_000}},
)
await handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=DualCache(),
data={**CHAT_BODY_FOR_RUST, "litellm_call_id": "rust-project-call"},
call_type="completion",
)
stash = get_request_stash()
assert stash is not None and stash.itpm_reserved_tokens == RUST_CHAT_INPUT_TOKENS
assert [json.loads(body) for body in rust_chat_factory.bodies] == [
{
"model": "gpt-4o",
"messages": CHAT_BODY_FOR_RUST["messages"],
"tools": CHAT_BODY_FOR_RUST["tools"],
"tool_choice": "auto",
}
]
@pytest.mark.asyncio
async def test_admission_input_tokens_fall_back_to_python_when_rust_declines(rust_chat_factory: _RustChatFactory):
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
rust_chat_factory.declined = True
audio_body = {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "transcribe"},
{
"type": "input_audio",
"input_audio": {"data": base64.b64encode(b"a" * 4_000).decode(), "format": "wav"},
},
],
}
],
}
python_estimate = handler._estimate_precise_input_tokens(audio_body, model="gpt-4o", call_type="acompletion")
estimate = await handler._estimate_admission_input_tokens(audio_body, model="gpt-4o", call_type="acompletion")
assert len(rust_chat_factory.bodies) == 1
assert estimate == python_estimate
assert estimate > handler._estimate_precise_input_tokens(
{"model": "gpt-4o", "messages": [{"role": "user", "content": "transcribe"}]},
model="gpt-4o",
call_type="acompletion",
)
@pytest.mark.asyncio
async def test_admission_input_tokens_stay_in_python_when_rust_is_disabled(rust_chat_factory: _RustChatFactory):
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
litellm.rust(False)
estimate = await handler._estimate_admission_input_tokens(
CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion"
)
assert rust_chat_factory.bodies == []
assert estimate == handler._estimate_precise_input_tokens(
CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("data", "call_type"),
(
({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, "aresponses"),
({"model": "gpt-4o", "input": ["hello"], "messages": [{"role": "user", "content": "ignored"}]}, "aembedding"),
(
{"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "ignored"}]},
"atext_completion",
),
(
{"model": "gpt-4o", "query": "q", "documents": ["d"], "messages": [{"role": "user", "content": "x"}]},
"arerank",
),
({"model": "gpt-4o", "contents": [{"role": "user", "parts": [{"text": "hello"}]}]}, "agenerate_content"),
({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, None),
({"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "both"}]}, "acompletion"),
({"model": "gpt-4o", "messages": "not a list"}, "acompletion"),
),
)
async def test_admission_input_tokens_keep_non_chat_shapes_in_python(
rust_chat_factory: _RustChatFactory, data: dict, call_type: str | None
):
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
estimate = await handler._estimate_admission_input_tokens(data, model="gpt-4o", call_type=call_type)
assert rust_chat_factory.bodies == []
assert estimate == handler._estimate_precise_input_tokens(data, model="gpt-4o", call_type=call_type)
@pytest.mark.asyncio
async def test_admission_input_tokens_keep_models_without_a_rust_tokenizer_in_python(
rust_chat_factory: _RustChatFactory, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(
litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"}
)
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
data = {**CHAT_BODY_FOR_RUST, "model": "text-davinci-003"}
estimate = await handler._estimate_admission_input_tokens(data, model="text-davinci-003", call_type="acompletion")
assert rust_chat_factory.bodies == []
assert estimate == handler._estimate_precise_input_tokens(data, model="text-davinci-003", call_type="acompletion")

View file

@ -304,6 +304,70 @@ def test_disabled_token_counter_declines_every_model(monkeypatch: pytest.MonkeyP
assert bridge.rust_tokenizer(O200K_MODEL) is None
CHAT_MESSAGES: Final = ({"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"})
CHAT_TOOLS: Final = ({"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}},)
@pytest.mark.asyncio
@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL))
async def test_chat_count_serializes_only_the_counted_fields(model: str) -> None:
factory: Final = _RecordingFactory()
litellm.rust(True)
bridge.TOKEN_COUNTER.override(factory)
count: Final = await bridge.count_chat_input_tokens(
model=model, messages=CHAT_MESSAGES, tools=CHAT_TOOLS, tool_choice="auto"
)
assert count == 42
assert [json.loads(body) for counter in factory.counters for body in counter.bodies] == [
{"model": model, "messages": list(CHAT_MESSAGES), "tools": list(CHAT_TOOLS), "tool_choice": "auto"}
]
@pytest.mark.asyncio
async def test_chat_count_declines_before_serializing_when_the_bridge_is_off() -> None:
factory: Final = _RecordingFactory()
litellm.rust(False)
bridge.TOKEN_COUNTER.override(factory)
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None
assert factory.counters == []
@pytest.mark.asyncio
async def test_chat_count_declines_models_without_a_rust_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"}
)
factory: Final = _RecordingFactory()
litellm.rust(True)
bridge.TOKEN_COUNTER.override(factory)
assert await bridge.count_chat_input_tokens(model="text-davinci-003", messages=CHAT_MESSAGES) is None
assert factory.counters == []
@pytest.mark.asyncio
async def test_chat_count_declines_when_the_native_counter_declines() -> None:
litellm.rust(True)
bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("unsupported content block")))
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None
@pytest.mark.asyncio
async def test_chat_count_declines_an_unserializable_body() -> None:
factory: Final = _RecordingFactory()
litellm.rust(True)
bridge.TOKEN_COUNTER.override(factory)
messages: Final = ({"role": "user", "content": object()},)
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None
assert factory.counters == []
PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = (
{"model": MODEL, "messages": [{"role": "user", "content": "Hello, how are you today?"}]},
{
@ -442,3 +506,45 @@ async def test_native_declines_shapes_python_prices_differently(
litellm.rust(True)
assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL))
@pytest.mark.parametrize(
"request_body", tuple(body for body in PARITY_REQUESTS if isinstance(body.get("messages"), list))
)
async def test_native_chat_count_matches_python_token_counter(
monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str
) -> None:
native: Final = pytest.importorskip("litellm.rust_bridge._native")
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
litellm.rust(True)
messages: Final = request_body["messages"]
assert isinstance(messages, list)
rust_count: Final = await bridge.count_chat_input_tokens(
model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice")
)
assert rust_count == litellm.token_counter(
model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice")
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"block",
(
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}},
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
),
)
async def test_native_chat_count_declines_multimodal_blocks(
monkeypatch: pytest.MonkeyPatch, block: dict[str, object]
) -> None:
native: Final = pytest.importorskip("litellm.rust_bridge._native")
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
litellm.rust(True)
messages: Final = ({"role": "user", "content": [{"type": "text", "text": "look"}, block]},)
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None