scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 71

Merge c8c9ef2690 into 5df502b360

Browse source
This commit is contained in:
Mihidum 2026-09-29 09:10:10 +10:00 • committed by GitHub
commit 889452e79c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 314 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
litellm/constants.py
View file

@ -85,6 +85,15 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLL
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
# Token estimate for one `input_audio` content block (audio understanding).
# The real cost is the provider's server-side audio tokenization and cannot be
# derived exactly client-side. With a base64 payload the estimate comes from
# the decoded byte count at a conservative low bitrate -- 8 kHz mono PCM-16
# (16 000 bytes/s) at 10 tokens/s -- so equal-duration higher-quality audio is
# never under-estimated; a payload-less (reference-only) block gets the flat
# per-block floor. parallel_request_limiter_v3 reserves with the same numbers.
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
AUDIO_BYTES_PER_TOKEN: Final = 1600
# Maximum wall-clock seconds a streaming response is allowed to run.
# Streams exceeding this duration are terminated with a Timeout error.

50
litellm/litellm_core_utils/token_counter.py
View file

@ -16,6 +16,8 @@ import litellm
from litellm import verbose_logger
from litellm._lazy_imports import _get_default_encoding
from litellm.constants import (
AUDIO_BYTES_PER_TOKEN,
DEFAULT_AUDIO_TOKEN_ESTIMATE,
DEFAULT_IMAGE_HEIGHT,
DEFAULT_IMAGE_TOKEN_COUNT,
DEFAULT_IMAGE_WIDTH,
@ -866,6 +868,47 @@ def _count_anthropic_content(
return tokens
def _count_input_audio_content_block(c: Mapping[str, object]) -> int:
"""
Estimate tokens for an OpenAI ``input_audio`` content block (audio
understanding), e.g. {"type": "input_audio", "input_audio": {"data":
"<base64>", "format": "wav"}}. The real token cost is the provider's
server-side audio tokenization and cannot be derived exactly client-side;
when the block carries a base64 payload, derive the estimate from the
decoded byte count at a conservative low bitrate, otherwise use the flat
per-block floor -- the same numbers ``parallel_request_limiter_v3``
already uses for its audio reservations (issue #38459).
"""
input_audio: Final = c.get("input_audio")
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
if isinstance(b64_data, str) and b64_data:
decoded_bytes: Final = len(b64_data) * 3 // 4
return max(decoded_bytes // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
return DEFAULT_AUDIO_TOKEN_ESTIMATE
def messages_contain_input_audio_content_blocks(messages: object) -> bool:
"""
True when any message carries an OpenAI ``input_audio`` content block.
Callers that use ``token_counter`` for reservations or rate limits must
check this first: an audio block's contribution is a size-derived
ESTIMATE at a deliberately low assumed bitrate (see
``_count_input_audio_content_block``), not a measurement, so the count
for such a message must not be trusted as an upper bound.
"""
if not isinstance(messages, list):
return False
for message in messages:
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_audio":
return True
return False
def _count_content_list(
count_function: TokenCounterFunction,
content_list: str
@ -921,8 +964,7 @@ def _count_content_list(
# Claude extended thinking content block
# Count the thinking text and skip the opaque blobs (signature, redacted data)
thinking_text = str(c.get("thinking", ""))
if thinking_text:
num_tokens += count_function(thinking_text)
num_tokens += count_function(thinking_text) if thinking_text else 0
elif c["type"] == "tool_reference":
# Anthropic tool-search reference block: a lightweight pointer to
# a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}.
@ -934,13 +976,15 @@ def _count_content_list(
tool_name = str(c.get("tool_name") or "")
if tool_name:
num_tokens += count_function(tool_name)
elif c["type"] == "input_audio":
num_tokens += _count_input_audio_content_block(c)
else:
content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__
raise ValueError(
f"Invalid content item type: {content_type}. "
f"Expected str or dict with 'type' field "
f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, "
f"tool_reference)."
f"tool_reference, input_audio)."
)
return num_tokens
except Exception as e:

15
litellm/proxy/hooks/batch_rate_limiter.py
View file

@ -37,6 +37,7 @@ from litellm.batches.batch_utils import (
from litellm.constants import BATCH_TPD_DESCRIPTOR_SUFFIX, BATCH_TPD_WINDOW_SECONDS
from litellm.exceptions import RateLimitErrorCategory
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.token_counter import messages_contain_input_audio_content_blocks
from litellm.proxy._types import (
ProxyErrorTypes,
ProxyException,
@ -971,6 +972,20 @@ class _PROXY_BatchRateLimiter(CustomLogger):
try:
entry_total_tokens = _count_entry_tokens(entry)
# An `input_audio` block's contribution is a size-derived
# estimate at a deliberately low assumed bitrate (see
# token_counter._count_input_audio_content_block), far
# below this fallback's raw-bytes estimate. Before such
# blocks were countable (#38459) an audio row RAISED
# inside token_counter and fell back to the size-based
# estimate; floor it there again so a row carrying a
# large base64 audio payload cannot slide the batch
# under the TPM limit.
entry_body = entry.get("body") if isinstance(entry, dict) else None
if isinstance(entry_body, dict) and messages_contain_input_audio_content_blocks(
entry_body.get("messages")
):
entry_total_tokens = max(entry_total_tokens, _estimate_batch_entry_tokens(raw_line))
except Exception:
entry_total_tokens = _estimate_batch_entry_tokens(raw_line)
total_tokens += entry_total_tokens

33
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -34,7 +34,12 @@ from typing_extensions import NotRequired, ReadOnly
from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.constants import (
AUDIO_BYTES_PER_TOKEN,
DEFAULT_AUDIO_TOKEN_ESTIMATE,
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
@ -446,18 +451,6 @@ GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = (
CallTypes.agenerate_content_stream.value,
)
RESPONSES_API_MIN_OUTPUT_TOKENS: Final = 16
# litellm.token_counter has no per-type handling for "input_audio" content
# blocks (unlike images, which use use_default_image_token_count) -- it
# silently contributes 0 tokens for them. When the block carries a base64
# payload, the estimate is derived from the decoded byte count; when the
# block is a reference without a payload (or the payload is missing), this
# flat per-block floor is used instead.
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
# Conservative bytes-per-token assumption for size-based audio estimation:
# equivalent to 8 kHz mono PCM-16 (16 000 bytes/s) at 10 tokens/s. Choosing
# the lowest reasonable bitrate means we never under-reserve for higher-
# quality audio recorded at the same wall-clock duration.
_AUDIO_BYTES_PER_TOKEN: Final = 1600
# Descriptor "key" values for project-scoped ITPM/OTPM. Distinct from
# "model_per_project" (the combined-TPM descriptor) so both can be enforced
# on the same project+model simultaneously without colliding on cache keys.
@ -3361,7 +3354,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
Token estimate for one ``input_audio`` content block.
When the block carries a base64 ``data`` payload, the estimate comes
from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``),
from the decoded byte count (``len(b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN``),
assuming the lowest reasonable audio bitrate so we never under-reserve
for higher-quality recordings of the same duration.
@ -3374,7 +3367,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
b64_data: Final = input_audio.get("data") if isinstance(input_audio, dict) else None
if b64_data and isinstance(b64_data, str):
decoded_bytes: Final = len(b64_data) * 3 // 4
return max(decoded_bytes // _AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
return max(decoded_bytes // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
return DEFAULT_AUDIO_TOKEN_ESTIMATE
@classmethod
@ -3400,11 +3393,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _strip_audio_content_blocks(messages: object) -> object:
"""
Drop ``input_audio`` content blocks before passing ``messages`` to
``token_counter``, which raises ``ValueError`` on them (no per-type
handling, unlike images). The audio contribution is added back
separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the
message (text/images/tools) still gets counted accurately instead of
the whole call falling back to the cheap char-count estimate.
``token_counter``. It counts them with the same size-derived estimate
since #38459 (it used to raise); stripping keeps the audio contribution
added exactly once, by ``_estimate_audio_content_tokens``, so the rest
of the message (text/images/tools) is counted accurately without ever
double-counting the audio.
"""
if not isinstance(messages, list):
return messages

18
litellm/proxy/spend_tracking/input_tokens.py
View file

@ -19,6 +19,7 @@ from typing import Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.token_counter import messages_contain_input_audio_content_blocks
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.token_counter import (
@ -128,15 +129,26 @@ def _count_input_tokens_for_models(
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
try:
if "messages" in request_body:
messages: Final = request_body.get("messages")
try:
return litellm.token_counter(
counted: Final = litellm.token_counter(
model=model,
messages=request_body.get("messages") or (),
messages=messages or (),
tools=request_body.get("tools"),
tool_choice=request_body.get("tool_choice"),
)
except ValueError:
return _count_text_tokens(model=model, text=request_body.get("messages"))
return _count_text_tokens(model=model, text=messages)
# An ``input_audio`` block counts as a size-derived estimate at a
# deliberately low assumed bitrate, and this reservation prices it
# at the text rate. Before such blocks were countable (#38459) an
# audio request RAISED in token_counter and reserved from the
# serialised-messages fallback above; floor it there again so a
# large or highly compressed audio payload cannot be admitted
# against a budget more cheaply than it was before.
if messages_contain_input_audio_content_blocks(messages):
return max(counted, _count_text_tokens(model=model, text=messages))
return counted
if "prompt" in request_body:
return _count_text_tokens(model=model, text=request_body.get("prompt"))
if "input" in request_body:

96
tests/test_litellm/proxy/hooks/test_batch_file_validation.py
View file

@ -2286,3 +2286,99 @@ async def test_disable_flag_still_skips_batch_processing_with_enqueued_limits():
assert result is data
afile_content_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_input_audio_row_reserves_size_based_floor():
"""A chat row carrying an OpenAI `input_audio` content block must reserve
at least the size-based estimate (serialized bytes / 4).
token_counter's audio contribution is a size-derived estimate at a
deliberately low assumed bitrate (decoded bytes / AUDIO_BYTES_PER_TOKEN),
far below the raw-bytes fallback. Before audio blocks were countable
(#38459) such a row RAISED inside token_counter and fell back to the
size-based estimate; the floor restores exactly that conservatism so a
large base64 audio payload cannot slide the batch under the TPM limit.
"""
import json as _json
from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter
prl = MagicMock()
prl.no_max_tokens_output_floor.return_value = 0
rate_limiter = _PROXY_BatchRateLimiter(
internal_usage_cache=MagicMock(),
parallel_request_limiter=prl,
)
blob = "A" * 400_000
audio_row_bytes = _json.dumps(
{
"custom_id": "row-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": "gpt-4o-audio-preview",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "What does the audio say?"},
{"type": "input_audio", "input_audio": {"data": blob, "format": "wav"}},
],
}
],
},
}
).encode("utf-8")
fake_content = MagicMock()
fake_content.content = audio_row_bytes
with patch( # test-quality-ok: mirrors the file's established harness — the download, not an HTTP boundary
"litellm.afile_content",
new=AsyncMock(return_value=fake_content),
):
usage = await rate_limiter.count_input_file_usage(
file_id="file-not-managed",
custom_llm_provider="openai",
user_api_key_dict=None,
)
size_based_floor = len(audio_row_bytes) // 4
assert usage.request_count == 1
assert usage.total_tokens >= size_based_floor, (
f"audio-block row must reserve at least the size-based estimate "
f"({size_based_floor} tokens for {len(audio_row_bytes)} bytes), got "
f"{usage.total_tokens} — a large audio payload would evade TPM limits"
)
# Control: a plain-text row must NOT be floored at its serialized size —
# measured text rows keep the (smaller) real token count.
text_row_bytes = _json.dumps(
{
"custom_id": "row-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": "gpt-4o-audio-preview",
"messages": [{"role": "user", "content": "What does the audio say?"}],
},
}
).encode("utf-8")
fake_text_content = MagicMock()
fake_text_content.content = text_row_bytes
with patch( # test-quality-ok: mirrors the file's established harness — the download, not an HTTP boundary
"litellm.afile_content",
new=AsyncMock(return_value=fake_text_content),
):
text_usage = await rate_limiter.count_input_file_usage(
file_id="file-not-managed",
custom_llm_provider="openai",
user_api_key_dict=None,
)
assert text_usage.request_count == 1
assert text_usage.total_tokens < len(text_row_bytes) // 4, (
"plain-text rows must keep the measured token count, not the size-based floor"
)

6
tests/test_litellm/proxy/hooks/test_tpm_concurrent.py
View file

@ -21,12 +21,12 @@ from typing import Any, Dict
import pytest
from litellm.caching.caching import DualCache
from litellm.constants import AUDIO_BYTES_PER_TOKEN
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROJECT_ITPM_DESCRIPTOR_KEY,
PROJECT_OTPM_DESCRIPTOR_KEY,
RateLimitedModel,
_AUDIO_BYTES_PER_TOKEN,
_PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler,
)
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
@ -2273,7 +2273,7 @@ def test_audio_token_estimate_scales_with_payload_size():
to exhaust ITPM quota while reserving almost nothing.
The estimate must now grow proportionally with the base64 payload size
(len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN), floored at
(len(b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN), floored at
DEFAULT_AUDIO_TOKEN_ESTIMATE so reference-only blocks and genuinely
short clips still get a non-trivial reservation.
@ -2311,7 +2311,7 @@ def test_audio_token_estimate_scales_with_payload_size():
f"Large payload ({large_estimate}) must reserve more than small payload "
f"({small_estimate}); flat-rate bug is back"
)
assert very_large_estimate == len(very_large_b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN
assert very_large_estimate == len(very_large_b64) * 3 // 4 // AUDIO_BYTES_PER_TOKEN
assert very_large_estimate > 6_000
assert no_data_estimate >= 300, (
f"Reference-only block (no data) must use the DEFAULT_AUDIO_TOKEN_ESTIMATE floor; got {no_data_estimate}"

39
tests/test_litellm/proxy/spend_tracking/test_input_tokens.py
View file

@ -2,6 +2,7 @@
from __future__ import annotations
import base64
from types import MappingProxyType
from typing import Final
@ -9,6 +10,7 @@ import pytest
from litellm.proxy.spend_tracking.input_tokens import (
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS,
_count_text_tokens,
count_input_tokens,
count_input_tokens_for_model,
)
@ -27,3 +29,40 @@ async def test_large_input_is_still_counted() -> None:
assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL)
assert isinstance(counts, MappingProxyType)
def test_input_audio_requests_reserve_at_least_the_serialised_fallback() -> None:
"""
Budget reservation counts an ``input_audio`` block as a size-derived
estimate at a deliberately low bitrate, priced at the text rate. Before
#38459 the same request raised inside ``token_counter`` and reserved
from the serialised-messages fallback, which tokenises the base64
payload itself. Floor audio-bearing requests at that fallback so a
caller cannot be admitted against a budget more cheaply than before
the blocks became countable (compressed audio carries far more duration
per byte than the estimate assumes).
"""
model: Final = "gpt-4o-audio-preview"
audio_b64: Final = base64.b64encode(bytes(range(256)) * 400).decode()
audio_messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe this recording."},
{"type": "input_audio", "input_audio": {"data": audio_b64, "format": "mp3"}},
],
}
]
text_messages: Final = [{"role": "user", "content": "Transcribe this recording."}]
audio_count: Final = count_input_tokens_for_model(request_body={"messages": audio_messages}, model=model)
fallback: Final = _count_text_tokens(model=model, text=audio_messages)
text_count: Final = count_input_tokens_for_model(request_body={"messages": text_messages}, model=model)
assert audio_count is not None, "an audio-bearing request must still be countable"
assert fallback > 0, "the serialised fallback must see the base64 payload"
assert audio_count >= fallback, (
f"audio request reserved {audio_count} tokens, below the pre-#38459 fallback of {fallback}"
)
assert text_count is not None
assert text_count < fallback, "a text-only request must not be floored"

77
tests/unit/litellm_core_utils/test_token_counter.py
View file

@ -1633,3 +1633,80 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t
"custom": expected["Xenova/llama-3-tokenizer"],
"requested": sorted(served),
}
def test_token_counter_with_input_audio_content_block():
"""
Regression test for issue #38459: a message containing an OpenAI
`input_audio` content block (audio understanding) must NOT raise from
token_counter. Before the fix the raise poisoned the whole message and
every caller that swallows counter errors failed open (router
context-window pre-call check, prompt-caching deployment check), while
/utils/token_counter returned HTTP 500.
The estimate mirrors parallel_request_limiter_v3's audio reservation:
decoded-base64 byte count at AUDIO_BYTES_PER_TOKEN, floored at
DEFAULT_AUDIO_TOKEN_ESTIMATE per block.
"""
from litellm.constants import AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE
small_b64 = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YQAAAAA="
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "What does the audio say?"},
{"type": "input_audio", "input_audio": {"data": small_b64, "format": "wav"}},
],
}
]
tokens = token_counter_new(model="gpt-4o-audio-preview", messages=messages)
assert tokens >= DEFAULT_AUDIO_TOKEN_ESTIMATE, f"Expected at least the per-block floor, got {tokens}"
# a large payload must scale the estimate (decoded bytes / AUDIO_BYTES_PER_TOKEN)
big_b64 = "A" * (AUDIO_BYTES_PER_TOKEN * 4000) # decoded ~3000 * AUDIO_BYTES_PER_TOKEN bytes
tokens_big = token_counter_new(
model="gpt-4o-audio-preview",
messages=[
{
"role": "user",
"content": [{"type": "input_audio", "input_audio": {"data": big_b64, "format": "wav"}}],
}
],
)
assert tokens_big >= 2900, f"large audio payload must scale the estimate, got {tokens_big}"
assert tokens_big > tokens
# a payload-less (reference-only) block must not raise and gets the floor
tokens_bare = token_counter_new(
model="gpt-4o-audio-preview",
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"format": "wav"}}]}],
)
assert tokens_bare >= DEFAULT_AUDIO_TOKEN_ESTIMATE
def test_trim_messages_with_input_audio_content_block():
"""Companion to issue #38459, same shape as the `file`-block case
(#28409): trim_messages swallows the token_counter error and silently
returns an over-budget conversation UNTRIMMED. With the fix, trimming
must actually happen."""
small_b64 = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YQAAAAA="
messages = [
{"role": "user", "content": "filler message " * 200},
{"role": "user", "content": "filler message " * 200},
{
"role": "user",
"content": [
{"type": "text", "text": "What does the audio say?"},
{"type": "input_audio", "input_audio": {"data": small_b64, "format": "wav"}},
],
},
]
trimmed = litellm.utils.trim_messages(messages, model="gpt-4o-audio-preview", max_tokens=500)
assert trimmed is not None
assert len(trimmed) < len(messages), (
"trim_messages must actually trim an over-budget conversation containing an input_audio block"
)