mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat(rust): count TPM admission tokens in Rust for plain chat requests
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
e4706fa409
commit
d4e2f2fff7
4 changed files with 366 additions and 2 deletions
|
|
@ -59,6 +59,7 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.rust_bridge.token_counter import count_chat_input_tokens
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -3283,6 +3284,42 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
|
||||
def _counts_messages_as_chat(self, data: Mapping[str, object], call_type: str | None) -> bool:
|
||||
return (
|
||||
data.get("prompt") is None
|
||||
and data.get("input") is None
|
||||
and call_type not in RESPONSES_API_CALL_TYPES
|
||||
and call_type not in RERANK_API_CALL_TYPES
|
||||
and call_type not in TEXT_COMPLETION_API_CALL_TYPES
|
||||
and call_type not in GOOGLE_GENAI_NATIVE_CALL_TYPES
|
||||
and not self._is_embedding_request(data, call_type)
|
||||
)
|
||||
|
||||
async def _estimate_admission_input_tokens(
|
||||
self, data: Mapping[str, object], model: str | None, call_type: str | None
|
||||
) -> int:
|
||||
"""
|
||||
``_estimate_precise_input_tokens`` with the Rust counter in front of it for plain chat bodies, the one
|
||||
shape whose Rust count equals ``token_counter``'s. Rust declines image and audio blocks, so those and
|
||||
every other API shape keep the Python path with its image defaults and audio add-on.
|
||||
"""
|
||||
messages: Final = data.get("messages")
|
||||
rust_count: Final = (
|
||||
await count_chat_input_tokens(
|
||||
model=model or "",
|
||||
messages=messages,
|
||||
tools=data.get("tools"),
|
||||
tool_choice=data.get("tool_choice"),
|
||||
)
|
||||
if isinstance(messages, list) and self._counts_messages_as_chat(data, call_type)
|
||||
else None
|
||||
)
|
||||
if rust_count is not None:
|
||||
return rust_count
|
||||
return await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=model, call_type=call_type
|
||||
)
|
||||
|
||||
async def _reserve_project_io_tokens_or_raise(
|
||||
self,
|
||||
descriptors: Sequence[RateLimitDescriptor],
|
||||
|
|
@ -3328,7 +3365,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
min_configured_tpm_limit=min_configured_otpm_limit,
|
||||
call_type=call_type,
|
||||
)
|
||||
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
raw_estimated_input_tokens: Final = await self._estimate_admission_input_tokens(
|
||||
data=data, model=requested_model, call_type=call_type
|
||||
)
|
||||
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables
|
||||
|
||||
import orjson
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
|
|
@ -112,3 +113,27 @@ async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputToke
|
|||
return None
|
||||
verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens)
|
||||
return attempt.value
|
||||
|
||||
|
||||
async def count_chat_input_tokens(
|
||||
model: str,
|
||||
messages: Sequence[object],
|
||||
tools: object = None,
|
||||
tool_choice: object = None,
|
||||
) -> int | None:
|
||||
"""Rust count of an already-parsed chat body, `None` when Python must count.
|
||||
|
||||
Callers holding Python messages instead of the raw request bytes pay one `orjson.dumps` of the counted fields.
|
||||
Pydantic messages and other non-JSON objects are left to Python, as are the blocks Rust declines."""
|
||||
if not rust_enabled() or TOKEN_COUNTER.load() is None:
|
||||
return None
|
||||
tokenizer: Final = rust_tokenizer(model)
|
||||
if tokenizer is None:
|
||||
return None
|
||||
try:
|
||||
body: Final = orjson.dumps({"model": model, "messages": messages, "tools": tools, "tool_choice": tool_choice})
|
||||
except orjson.JSONEncodeError as error:
|
||||
verbose_logger.debug("Rust token counter (%s) skipped an unserializable body: %s", tokenizer, error)
|
||||
return None
|
||||
count: Final = await count_input_tokens(body, tokenizer)
|
||||
return None if count is None else count.input_tokens
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ Unit Tests for the max parallel request limiter v3 for the proxy
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -6284,3 +6286,197 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_
|
|||
assert isinstance(values, list)
|
||||
assert [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING] == []
|
||||
assert any("circuit breaker is open" in record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
class _FakeRustDeclined(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeRustUpstream(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeRustNative:
|
||||
RustBridgeDeclined = _FakeRustDeclined
|
||||
RustUpstreamError = _FakeRustUpstream
|
||||
|
||||
|
||||
RUST_CHAT_INPUT_TOKENS = 777
|
||||
|
||||
|
||||
class _RustChatCounter:
|
||||
def __init__(self, bodies: list[bytes], declined: bool) -> None:
|
||||
self.bodies = bodies
|
||||
self.declined = declined
|
||||
|
||||
async def acount_request(self, body: bytes) -> object:
|
||||
self.bodies.append(body)
|
||||
if self.declined:
|
||||
raise _FakeRustDeclined("unsupported content block")
|
||||
return {"model": "", "input_tokens": RUST_CHAT_INPUT_TOKENS}
|
||||
|
||||
|
||||
class _RustChatFactory:
|
||||
"""Stands in for the native `TokenCounter` class and records every body handed to any tokenizer."""
|
||||
|
||||
def __init__(self, declined: bool = False) -> None:
|
||||
self.bodies: list[bytes] = []
|
||||
self.declined = declined
|
||||
|
||||
def __call__(self, tokenizer_json: str) -> _RustChatCounter:
|
||||
return _RustChatCounter(self.bodies, self.declined)
|
||||
|
||||
def from_cl100k_ranks(self, rank_file: str) -> _RustChatCounter:
|
||||
return _RustChatCounter(self.bodies, self.declined)
|
||||
|
||||
def from_o200k_ranks(self, rank_file: str) -> _RustChatCounter:
|
||||
return _RustChatCounter(self.bodies, self.declined)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rust_chat_factory(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.rust_bridge import bindings, configuration
|
||||
from litellm.rust_bridge import token_counter as rust_token_counter
|
||||
|
||||
monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeRustNative())
|
||||
rust_token_counter._counter.cache_clear()
|
||||
configuration.reset_rust_configuration()
|
||||
factory = _RustChatFactory()
|
||||
litellm.rust(True)
|
||||
rust_token_counter.TOKEN_COUNTER.override(factory)
|
||||
yield factory
|
||||
rust_token_counter.TOKEN_COUNTER.reset()
|
||||
rust_token_counter._counter.cache_clear()
|
||||
configuration.reset_rust_configuration()
|
||||
|
||||
|
||||
CHAT_BODY_FOR_RUST = {
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"}],
|
||||
"tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}],
|
||||
"tool_choice": "auto",
|
||||
"max_tokens": 50,
|
||||
"metadata": {"user_api_key": "sk-hidden"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_project_itpm_reservation_counts_plain_chat_bodies_in_rust(rust_chat_factory: _RustChatFactory):
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
user_api_key_dict = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-rust-itpm"),
|
||||
project_id="proj-rust",
|
||||
project_metadata={"model_itpm_limit": {"gpt-4o": 10_000}},
|
||||
)
|
||||
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
cache=DualCache(),
|
||||
data={**CHAT_BODY_FOR_RUST, "litellm_call_id": "rust-project-call"},
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
stash = get_request_stash()
|
||||
assert stash is not None and stash.itpm_reserved_tokens == RUST_CHAT_INPUT_TOKENS
|
||||
assert [json.loads(body) for body in rust_chat_factory.bodies] == [
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"messages": CHAT_BODY_FOR_RUST["messages"],
|
||||
"tools": CHAT_BODY_FOR_RUST["tools"],
|
||||
"tool_choice": "auto",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admission_input_tokens_fall_back_to_python_when_rust_declines(rust_chat_factory: _RustChatFactory):
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
rust_chat_factory.declined = True
|
||||
audio_body = {
|
||||
"model": "gpt-4o",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "transcribe"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": base64.b64encode(b"a" * 4_000).decode(), "format": "wav"},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
python_estimate = handler._estimate_precise_input_tokens(audio_body, model="gpt-4o", call_type="acompletion")
|
||||
|
||||
estimate = await handler._estimate_admission_input_tokens(audio_body, model="gpt-4o", call_type="acompletion")
|
||||
|
||||
assert len(rust_chat_factory.bodies) == 1
|
||||
assert estimate == python_estimate
|
||||
assert estimate > handler._estimate_precise_input_tokens(
|
||||
{"model": "gpt-4o", "messages": [{"role": "user", "content": "transcribe"}]},
|
||||
model="gpt-4o",
|
||||
call_type="acompletion",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admission_input_tokens_stay_in_python_when_rust_is_disabled(rust_chat_factory: _RustChatFactory):
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
litellm.rust(False)
|
||||
|
||||
estimate = await handler._estimate_admission_input_tokens(
|
||||
CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion"
|
||||
)
|
||||
|
||||
assert rust_chat_factory.bodies == []
|
||||
assert estimate == handler._estimate_precise_input_tokens(
|
||||
CHAT_BODY_FOR_RUST, model="gpt-4o", call_type="acompletion"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("data", "call_type"),
|
||||
(
|
||||
({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, "aresponses"),
|
||||
({"model": "gpt-4o", "input": ["hello"], "messages": [{"role": "user", "content": "ignored"}]}, "aembedding"),
|
||||
(
|
||||
{"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "ignored"}]},
|
||||
"atext_completion",
|
||||
),
|
||||
(
|
||||
{"model": "gpt-4o", "query": "q", "documents": ["d"], "messages": [{"role": "user", "content": "x"}]},
|
||||
"arerank",
|
||||
),
|
||||
({"model": "gpt-4o", "contents": [{"role": "user", "parts": [{"text": "hello"}]}]}, "agenerate_content"),
|
||||
({"model": "gpt-4o", "input": "hello", "messages": [{"role": "user", "content": "ignored"}]}, None),
|
||||
({"model": "gpt-4o", "prompt": "hello", "messages": [{"role": "user", "content": "both"}]}, "acompletion"),
|
||||
({"model": "gpt-4o", "messages": "not a list"}, "acompletion"),
|
||||
),
|
||||
)
|
||||
async def test_admission_input_tokens_keep_non_chat_shapes_in_python(
|
||||
rust_chat_factory: _RustChatFactory, data: dict, call_type: str | None
|
||||
):
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
|
||||
estimate = await handler._estimate_admission_input_tokens(data, model="gpt-4o", call_type=call_type)
|
||||
|
||||
assert rust_chat_factory.bodies == []
|
||||
assert estimate == handler._estimate_precise_input_tokens(data, model="gpt-4o", call_type=call_type)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admission_input_tokens_keep_models_without_a_rust_tokenizer_in_python(
|
||||
rust_chat_factory: _RustChatFactory, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"}
|
||||
)
|
||||
handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
|
||||
data = {**CHAT_BODY_FOR_RUST, "model": "text-davinci-003"}
|
||||
|
||||
estimate = await handler._estimate_admission_input_tokens(data, model="text-davinci-003", call_type="acompletion")
|
||||
|
||||
assert rust_chat_factory.bodies == []
|
||||
assert estimate == handler._estimate_precise_input_tokens(data, model="text-davinci-003", call_type="acompletion")
|
||||
|
|
|
|||
|
|
@ -304,6 +304,70 @@ def test_disabled_token_counter_declines_every_model(monkeypatch: pytest.MonkeyP
|
|||
assert bridge.rust_tokenizer(O200K_MODEL) is None
|
||||
|
||||
|
||||
CHAT_MESSAGES: Final = ({"role": "system", "content": "be terse"}, {"role": "user", "content": "hello"})
|
||||
CHAT_TOOLS: Final = ({"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL))
|
||||
async def test_chat_count_serializes_only_the_counted_fields(model: str) -> None:
|
||||
factory: Final = _RecordingFactory()
|
||||
litellm.rust(True)
|
||||
bridge.TOKEN_COUNTER.override(factory)
|
||||
|
||||
count: Final = await bridge.count_chat_input_tokens(
|
||||
model=model, messages=CHAT_MESSAGES, tools=CHAT_TOOLS, tool_choice="auto"
|
||||
)
|
||||
|
||||
assert count == 42
|
||||
assert [json.loads(body) for counter in factory.counters for body in counter.bodies] == [
|
||||
{"model": model, "messages": list(CHAT_MESSAGES), "tools": list(CHAT_TOOLS), "tool_choice": "auto"}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_count_declines_before_serializing_when_the_bridge_is_off() -> None:
|
||||
factory: Final = _RecordingFactory()
|
||||
litellm.rust(False)
|
||||
bridge.TOKEN_COUNTER.override(factory)
|
||||
|
||||
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None
|
||||
assert factory.counters == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_count_declines_models_without_a_rust_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"}
|
||||
)
|
||||
factory: Final = _RecordingFactory()
|
||||
litellm.rust(True)
|
||||
bridge.TOKEN_COUNTER.override(factory)
|
||||
|
||||
assert await bridge.count_chat_input_tokens(model="text-davinci-003", messages=CHAT_MESSAGES) is None
|
||||
assert factory.counters == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_count_declines_when_the_native_counter_declines() -> None:
|
||||
litellm.rust(True)
|
||||
bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("unsupported content block")))
|
||||
|
||||
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=CHAT_MESSAGES) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_count_declines_an_unserializable_body() -> None:
|
||||
factory: Final = _RecordingFactory()
|
||||
litellm.rust(True)
|
||||
bridge.TOKEN_COUNTER.override(factory)
|
||||
|
||||
messages: Final = ({"role": "user", "content": object()},)
|
||||
|
||||
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None
|
||||
assert factory.counters == []
|
||||
|
||||
|
||||
PARITY_REQUESTS: Final[tuple[dict[str, object], ...]] = (
|
||||
{"model": MODEL, "messages": [{"role": "user", "content": "Hello, how are you today?"}]},
|
||||
{
|
||||
|
|
@ -442,3 +506,45 @@ async def test_native_declines_shapes_python_prices_differently(
|
|||
litellm.rust(True)
|
||||
|
||||
assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", (MODEL, CL100K_MODEL, O200K_MODEL))
|
||||
@pytest.mark.parametrize(
|
||||
"request_body", tuple(body for body in PARITY_REQUESTS if isinstance(body.get("messages"), list))
|
||||
)
|
||||
async def test_native_chat_count_matches_python_token_counter(
|
||||
monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str
|
||||
) -> None:
|
||||
native: Final = pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
|
||||
litellm.rust(True)
|
||||
messages: Final = request_body["messages"]
|
||||
assert isinstance(messages, list)
|
||||
|
||||
rust_count: Final = await bridge.count_chat_input_tokens(
|
||||
model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice")
|
||||
)
|
||||
|
||||
assert rust_count == litellm.token_counter(
|
||||
model=model, messages=messages, tools=request_body.get("tools"), tool_choice=request_body.get("tool_choice")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"block",
|
||||
(
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}},
|
||||
{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
|
||||
),
|
||||
)
|
||||
async def test_native_chat_count_declines_multimodal_blocks(
|
||||
monkeypatch: pytest.MonkeyPatch, block: dict[str, object]
|
||||
) -> None:
|
||||
native: Final = pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setattr(bindings, "get_native_bridge", lambda: native)
|
||||
litellm.rust(True)
|
||||
messages: Final = ({"role": "user", "content": [{"type": "text", "text": "look"}, block]},)
|
||||
|
||||
assert await bridge.count_chat_input_tokens(model=O200K_MODEL, messages=messages) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue