From ede726d46567928c211de10e21b1ce5cfffdcb74 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:50:27 +1000 Subject: [PATCH 1/4] fix(token_counter): support OpenAI 'file' content blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit token_counter raised 'Invalid content item type: file' on document understanding payloads ({'type': 'file', 'file': {...}}). On 1.83.14 this crashed trim_messages outright; current versions swallow the error and silently return the conversation UNTRIMMED, so over-budget conversations containing a file block are never trimmed. A file's real token cost is the provider's server-side extraction and cannot be derived client-side, and counting the opaque base64 file_data blob would wildly overcount — so count only the lightweight textual fields (file_id / filename / format), mirroring how tool_reference blocks count just the tool name. Fixes #28409 Co-Authored-By: Claude Fable 5 --- litellm/litellm_core_utils/token_counter.py | 24 +++++- .../litellm_core_utils/test_token_counter.py | 86 +++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 858b078d626..a2ca09ca7af 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -695,6 +695,26 @@ def _count_anthropic_content( return tokens +def _count_file_content_block(count_function: TokenCounterFunction, c: Mapping[str, object]) -> int: + """ + Count an OpenAI file content block (document understanding), e.g. + {"type": "file", "file": {"file_id" | "file_data" | "filename", ...}}. + The real token cost is the provider's server-side extraction of the + document and cannot be derived client-side; counting the base64 file_data + blob would wildly overcount. Count only the lightweight textual fields so + token_counter/trim_messages accept valid payloads instead of raising + (issue #28409). + """ + num_tokens = 0 + file_obj = c.get("file") + if isinstance(file_obj, dict): + for file_field in ("file_id", "filename", "format"): + file_field_value = file_obj.get(file_field) + if isinstance(file_field_value, str) and file_field_value: + num_tokens += count_function(file_field_value) + return num_tokens + + def _count_content_list( count_function: TokenCounterFunction, content_list: OpenAIMessageContent, @@ -738,11 +758,13 @@ def _count_content_list( tool_name = str(c.get("tool_name") or "") if tool_name: num_tokens += count_function(tool_name) + elif c["type"] == "file": + num_tokens += _count_file_content_block(count_function, c) else: content_type = c.get("type", type(c).__name__) if isinstance(c, dict) else type(c).__name__ raise ValueError( f"Invalid content item type: {content_type}. " - f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference)." + f"Expected str or dict with 'type' field (text, image_url, tool_use, tool_result, thinking, tool_reference, file)." ) return num_tokens except Exception as e: diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index a2590dbca2d..141b0ab7fa3 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1160,3 +1160,89 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + + +def test_token_counter_with_file_content_block(): + """ + Regression test for issue #28409: a message containing an OpenAI `file` + content block (document understanding) must NOT raise from token_counter + or trim_messages. + + The real token cost of a file is the provider's server-side extraction and + cannot be derived client-side; only the lightweight textual fields + (file_id / filename / format) are counted — the opaque base64 `file_data` + blob must not inflate the count. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document."}, + { + "type": "file", + "file": { + "file_id": "file-abc123", + "filename": "report.pdf", + "format": "application/pdf", + }, + }, + ], + } + ] + + tokens = token_counter_new(model="gpt-4o", messages=messages) + assert tokens > 0, f"Expected positive token count, got {tokens}" + + # base64 file_data is opaque — it must not blow up the count + small_blob = dict(messages[0]["content"][1]["file"], file_data="data:application/pdf;base64,AAAA") + big_blob = dict(messages[0]["content"][1]["file"], file_data="data:application/pdf;base64," + "A" * 100_000) + tokens_small = token_counter_new( + model="gpt-4o", + messages=[{"role": "user", "content": [{"type": "file", "file": small_blob}]}], + ) + tokens_big = token_counter_new( + model="gpt-4o", + messages=[{"role": "user", "content": [{"type": "file", "file": big_blob}]}], + ) + assert tokens_small == tokens_big, "opaque file_data blob must not be counted" + + # a bare file block (e.g. only file_data) must also not raise + tokens_bare = token_counter_new( + model="gpt-4o", + messages=[ + { + "role": "user", + "content": [{"type": "file", "file": {"file_data": "data:application/pdf;base64,AAAA"}}], + } + ], + ) + assert tokens_bare >= 0 + + +def test_trim_messages_with_file_content_block(): + """The original repro from issue #28409: trim_messages on a document + understanding payload raised ValueError from token_counter (1.83.14) — + current versions swallow that error and silently return the messages + UNTRIMMED instead. With the fix, trimming must actually happen.""" + messages = [ + {"role": "user", "content": "filler message " * 200}, + {"role": "user", "content": "filler message " * 200}, + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this file?"}, + { + "type": "file", + "file": {"file_id": "file-abc123", "filename": "report.pdf"}, + }, + ], + }, + ] + + trimmed = litellm.utils.trim_messages(messages, model="gpt-4o", max_tokens=120) + + assert trimmed is not None + assert len(trimmed) < len(messages), ( + "trim_messages must actually trim an over-budget conversation containing " + f"a file content block — got {len(trimmed)} messages back (input {len(messages)})" + ) From 694dd513f84ee6c2978e2177c409519152d6d769 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:51:26 +1000 Subject: [PATCH 2/4] fix(budget_reservation): reserve max_input_tokens for file content blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file block's real token cost is the provider's server-side document extraction — token_counter now (correctly) returns only the lightweight filename/id tokens, but budget reservation previously relied on the counter raising to hit its conservative max_input_tokens fallback. Detect file blocks explicitly and keep the conservative reservation. Also assert 'file' appears in the unknown-type error enumeration. Co-Authored-By: Claude Fable 5 --- .../spend_tracking/budget_reservation.py | 32 ++++++++++++--- .../litellm_core_utils/test_token_counter.py | 1 + .../proxy/test_budget_reservation.py | 39 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ce6c9330620..bb16a5c419b 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1273,12 +1273,19 @@ def _approximate_input_size(request_body: dict) -> int: def _count_input_tokens(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: - return litellm.token_counter( - model=model, - messages=request_body.get("messages") or [], - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) + messages = request_body.get("messages") or [] + # A `file` content block's real token cost is the provider's + # server-side extraction of the referenced document and cannot be + # derived client-side — token_counter only counts its filename/id + # fields, which would under-reserve arbitrarily large uploads. + # Fall through to the conservative max_input_tokens reservation. + if not _messages_contain_file_content_blocks(messages): + return litellm.token_counter( + model=model, + messages=messages, + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) if "prompt" in request_body: return _count_text_tokens(model=model, text=request_body.get("prompt")) if "input" in request_body: @@ -1315,6 +1322,19 @@ def _estimate_input_tokens( return None +def _messages_contain_file_content_blocks(messages: object) -> bool: + if not isinstance(messages, list): + return False + for message in messages: + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + continue + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "file": + return True + return False + + DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK: Final = 16384 diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 141b0ab7fa3..e116c39edcc 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -1160,6 +1160,7 @@ def test_count_content_list_rejects_unknown_type(): message = str(exc_info.value) assert "Invalid content item type: totally_unknown_block" in message assert "tool_reference" in message + assert "file" in message def test_token_counter_with_file_content_block(): diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 2388654bf4b..becc972d9f5 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -1108,6 +1108,45 @@ def test_reservation_uses_most_expensive_deployment_in_group(): assert estimated == pytest.approx(expected_expensive) +def test_estimate_input_tokens_reserves_max_for_file_content_blocks(): + """A `file` content block's real token cost is the provider's server-side + extraction of the referenced document — token_counter only sees the + filename/id, so counting it would under-reserve arbitrarily large uploads. + Reservation must fall back to the conservative max_input_tokens instead.""" + from litellm.proxy.spend_tracking.budget_reservation import ( + _estimate_input_tokens, + ) + + file_request = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document."}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + } + estimated = _estimate_input_tokens( + request_body=file_request, + route="/chat/completions", + model="gpt-4o", + model_info={"max_input_tokens": 128000}, + ) + assert estimated == 128000 + + # Plain text messages must still be counted, not blanket-reserved. + text_request = {"messages": [{"role": "user", "content": "hi"}]} + counted = _estimate_input_tokens( + request_body=text_request, + route="/chat/completions", + model="gpt-4o", + model_info={"max_input_tokens": 128000}, + ) + assert counted is not None and 0 < counted < 128000 + + @pytest.mark.asyncio async def test_should_clamp_reservation_to_model_ceiling_when_caller_overrequests( spend_counter_state, From 28ab28eb041b1373f3c4c2245dde8ebd107d3ea8 Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:33:44 +1000 Subject: [PATCH 3/4] fix(budget_reservation): return None explicitly when messages carry file blocks Skipping token_counter without returning let control fall through to the prompt / input / query branches, so a stray text field on a chat body with a file block was counted in place of the conservative max_input_tokens reservation. Return None from the messages branch instead; the decoy case is now pinned by a test. --- .../spend_tracking/budget_reservation.py | 20 +++++++++++-------- .../proxy/test_budget_reservation.py | 11 ++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index bb16a5c419b..ad9266dfb95 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1278,14 +1278,18 @@ def _count_input_tokens(request_body: dict, model: str) -> int | None: # server-side extraction of the referenced document and cannot be # derived client-side — token_counter only counts its filename/id # fields, which would under-reserve arbitrarily large uploads. - # Fall through to the conservative max_input_tokens reservation. - if not _messages_contain_file_content_blocks(messages): - return litellm.token_counter( - model=model, - messages=messages, - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) + # Return None so the caller takes the conservative max_input_tokens + # reservation — an explicit return, because falling through would + # let a stray `prompt` / `input` / `query` on the same body count + # as the whole request. + if _messages_contain_file_content_blocks(messages): + return None + return litellm.token_counter( + model=model, + messages=messages, + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) if "prompt" in request_body: return _count_text_tokens(model=model, text=request_body.get("prompt")) if "input" in request_body: diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index becc972d9f5..6f971c706f3 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -1136,6 +1136,17 @@ def test_estimate_input_tokens_reserves_max_for_file_content_blocks(): ) assert estimated == 128000 + # A stray text field on the same body must not be counted in place of the + # file — the messages branch owns the request once `messages` is present. + decoy_request = {**file_request, "prompt": "hi"} + decoy_estimated = _estimate_input_tokens( + request_body=decoy_request, + route="/chat/completions", + model="gpt-4o", + model_info={"max_input_tokens": 128000}, + ) + assert decoy_estimated == 128000 + # Plain text messages must still be counted, not blanket-reserved. text_request = {"messages": [{"role": "user", "content": "hi"}]} counted = _estimate_input_tokens( From 9a46ab3483970024058cd9befc2c9a1f11ed4cad Mon Sep 17 00:00:00 2001 From: Mihidum Hettiyahandi <55163074+mihidumh@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:53:31 +1000 Subject: [PATCH 4/4] fix(batch): floor file-block rows at the size-based estimate so file payloads cannot evade TPM limits Before file blocks were countable, a batch row carrying one RAISED inside token_counter and the batch rate limiter fell back to its conservative size-based estimate (raw bytes / 4), which covers the base64 payload. Making the row countable replaced that with the counter's few-token result (only filename/id fields are measurable client-side), so a row carrying a large base64 `file_data` reserved ~nothing and a crafted batch could slide under the TPM limit. Restore the conservatism at the rate-limiter call site: when a row's messages carry a `file` content block, take max(counted, size-based estimate). Plain rows keep the measured count. The `_messages_contain_file_content_blocks` helper moves to token_counter.py next to `_count_file_content_block` so both consumers (budget reservation, batch rate limiter) share it. --- litellm/litellm_core_utils/token_counter.py | 26 ++++- litellm/proxy/hooks/batch_rate_limiter.py | 10 ++ .../spend_tracking/budget_reservation.py | 18 +--- .../proxy/hooks/test_batch_file_validation.py | 99 +++++++++++++++++++ 4 files changed, 136 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index a2ca09ca7af..9ce50cac005 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -705,8 +705,8 @@ def _count_file_content_block(count_function: TokenCounterFunction, c: Mapping[s token_counter/trim_messages accept valid payloads instead of raising (issue #28409). """ - num_tokens = 0 - file_obj = c.get("file") + num_tokens = 0 # rebind-ok: accumulates over the file fields below + file_obj: Final = c.get("file") if isinstance(file_obj, dict): for file_field in ("file_id", "filename", "format"): file_field_value = file_obj.get(file_field) @@ -715,6 +715,28 @@ def _count_file_content_block(count_function: TokenCounterFunction, c: Mapping[s return num_tokens +def messages_contain_file_content_blocks(messages: object) -> bool: + """ + True when any message carries an OpenAI ``file`` content block. + + Callers that use ``token_counter`` for reservations or rate limits must + check this first: a file block's document payload is opaque to the counter + (only the ``file_id``/``filename``/``format`` fields are counted -- see + ``_count_file_content_block``), so the count for such a message is a floor, + not a measurement, and it must not be trusted as an upper bound. + """ + if not isinstance(messages, list): + return False + for message in messages: + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, list): + continue + for content_item in content: + if isinstance(content_item, dict) and content_item.get("type") == "file": + return True + return False + + def _count_content_list( count_function: TokenCounterFunction, content_list: OpenAIMessageContent, diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 5b814ad28fd..8672580e194 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -35,6 +35,7 @@ from litellm.batches.batch_utils import ( ) from litellm.exceptions import RateLimitErrorCategory from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.token_counter import messages_contain_file_content_blocks from litellm.proxy._types import ( ProxyErrorTypes, ProxyException, @@ -853,6 +854,15 @@ class _PROXY_BatchRateLimiter(CustomLogger): try: entry_total_tokens = _count_entry_tokens(entry) + # A `file` content block's document payload is opaque to + # token_counter -- only its filename/id fields are counted + # (see token_counter._count_file_content_block) -- so a row + # carrying a large base64 `file_data` would count as a few + # tokens and slide the whole batch under the TPM limit. + # Floor such rows at the size-based estimate, which does + # cover the payload bytes. + if messages_contain_file_content_blocks((entry.get("body") or {}).get("messages")): + entry_total_tokens = max(entry_total_tokens, _estimate_batch_entry_tokens(raw_line)) except Exception: entry_total_tokens = _estimate_batch_entry_tokens(raw_line) total_tokens += entry_total_tokens diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index ad9266dfb95..2afa0055faf 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate +from litellm.litellm_core_utils.token_counter import messages_contain_file_content_blocks from litellm.proxy._types import ( Litellm_EntityType, LiteLLM_TeamMembership, @@ -1273,7 +1274,7 @@ def _approximate_input_size(request_body: dict) -> int: def _count_input_tokens(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: - messages = request_body.get("messages") or [] + messages: Final = request_body.get("messages") or [] # A `file` content block's real token cost is the provider's # server-side extraction of the referenced document and cannot be # derived client-side — token_counter only counts its filename/id @@ -1282,7 +1283,7 @@ def _count_input_tokens(request_body: dict, model: str) -> int | None: # reservation — an explicit return, because falling through would # let a stray `prompt` / `input` / `query` on the same body count # as the whole request. - if _messages_contain_file_content_blocks(messages): + if messages_contain_file_content_blocks(messages): return None return litellm.token_counter( model=model, @@ -1326,19 +1327,6 @@ def _estimate_input_tokens( return None -def _messages_contain_file_content_blocks(messages: object) -> bool: - if not isinstance(messages, list): - return False - for message in messages: - content = message.get("content") if isinstance(message, dict) else None - if not isinstance(content, list): - continue - for content_item in content: - if isinstance(content_item, dict) and content_item.get("type") == "file": - return True - return False - - DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK: Final = 16384 diff --git a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py index 4ef94f0965b..09f468d1f5c 100644 --- a/tests/test_litellm/proxy/hooks/test_batch_file_validation.py +++ b/tests/test_litellm/proxy/hooks/test_batch_file_validation.py @@ -1827,6 +1827,105 @@ async def test_count_input_file_usage_streams_without_building_list(): mock_dict_list.assert_not_called() +@pytest.mark.asyncio +async def test_file_content_block_row_reserves_size_based_floor(): + """A chat row carrying an OpenAI `file` content block must reserve at + least the size-based estimate (serialized bytes / 4). + + token_counter deliberately counts only the block's filename/id fields — + the document payload is opaque to it — so without this floor a row + carrying a large base64 `file_data` counts as a handful of tokens and the + whole batch slides under the TPM limit. Before file blocks were countable + (#33659) such a row RAISED inside token_counter and fell back to the + size-based estimate; the floor restores exactly that conservatism. + """ + import json as _json + + from litellm.proxy.hooks.batch_rate_limiter import _PROXY_BatchRateLimiter + + prl = MagicMock() + prl.no_max_tokens_output_floor.return_value = 0 + rate_limiter = _PROXY_BatchRateLimiter( + internal_usage_cache=MagicMock(), + parallel_request_limiter=prl, + ) + + blob = "data:application/pdf;base64," + "A" * 400_000 + file_row_bytes = _json.dumps( + { + "custom_id": "row-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gpt-4o-mini", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document."}, + { + "type": "file", + "file": {"filename": "report.pdf", "file_data": blob}, + }, + ], + } + ], + }, + } + ).encode("utf-8") + fake_content = MagicMock() + fake_content.content = file_row_bytes + + with patch( # test-quality-ok: mirrors the file's established harness — the download, not an HTTP boundary + "litellm.afile_content", + new=AsyncMock(return_value=fake_content), + ): + usage = await rate_limiter.count_input_file_usage( + file_id="file-not-managed", + custom_llm_provider="openai", + user_api_key_dict=None, + ) + + size_based_floor = len(file_row_bytes) // 4 + assert usage.request_count == 1 + assert usage.total_tokens >= size_based_floor, ( + f"file-block row must reserve at least the size-based estimate " + f"({size_based_floor} tokens for {len(file_row_bytes)} bytes), got " + f"{usage.total_tokens} — a large file_data payload would evade TPM limits" + ) + + # Control: a plain-text row must NOT be floored at its serialized size — + # measured text rows keep the (smaller) real token count. + text_row_bytes = _json.dumps( + { + "custom_id": "row-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "Summarize this document."}], + }, + } + ).encode("utf-8") + fake_text_content = MagicMock() + fake_text_content.content = text_row_bytes + + with patch( # test-quality-ok: mirrors the file's established harness — the download, not an HTTP boundary + "litellm.afile_content", + new=AsyncMock(return_value=fake_text_content), + ): + text_usage = await rate_limiter.count_input_file_usage( + file_id="file-not-managed", + custom_llm_provider="openai", + user_api_key_dict=None, + ) + + assert 0 < text_usage.total_tokens < len(text_row_bytes) // 4, ( + "plain-text rows must keep the measured token count, not the size floor " + f"— got {text_usage.total_tokens} for a {len(text_row_bytes)}-byte row" + ) + + def _one_row_batch_bytes(model: str) -> bytes: import json as _json