From 20dcfde0799e0d6077bdc8a5a1a2d05cb36c0bdc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:34:48 -0700 Subject: [PATCH] fix(proxy): count a team's wildcard route the way routing serves it and reserve OpenAI chat's per-PDF charge --- litellm/constants.py | 1 + .../hooks/parallel_request_limiter_v3.py | 11 +++- .../spend_tracking/budget_reservation.py | 18 ++++- litellm/proxy/spend_tracking/input_tokens.py | 65 +++++++++++++++++-- .../hooks/test_parallel_request_limiter_v3.py | 24 +++++++ .../spend_tracking/test_budget_reservation.py | 27 ++++++++ .../proxy/spend_tracking/test_input_tokens.py | 21 ++++++ 7 files changed, 158 insertions(+), 9 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index ef70f1c8163..846593591da 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -753,6 +753,7 @@ ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 +OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000 diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index edbb4defafb..d3666ed0873 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,7 +76,11 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit -from litellm.proxy.spend_tracking.input_tokens import models_counted_for +from litellm.proxy.spend_tracking.input_tokens import ( + inline_pdf_file_count, + models_counted_for, + openai_chat_pdf_file_tokens, +) from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -514,6 +518,7 @@ _TPM_FLOOR_FRACTION: Final = 4 # check on data["input"] must exclude these call types, or a Responses call # gets misclassified as an embedding and skips output-token reservation/caps. RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses") +CHAT_COMPLETION_API_CALL_TYPES: Final = (CallTypes.completion.value, CallTypes.acompletion.value) EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding") TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion") RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value) @@ -4016,6 +4021,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages + chat_pdf_files: Final = ( + inline_pdf_file_count(countable_messages) if call_type in CHAT_COMPLETION_API_CALL_TYPES else 0 + ) try: estimate: Final = max( @@ -4031,6 +4039,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): use_default_image_token_count=True, ) ) + + openai_chat_pdf_file_tokens(tokenizer_model, chat_pdf_files) for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",)) ), ) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 3f3256a059b..c8c28f108c3 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -17,6 +17,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.proxy._types import ( @@ -43,13 +44,16 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.spend_tracking.input_tokens import ( count_input_tokens, count_input_tokens_for_model, + inline_pdf_file_count, models_counted_for, + openai_chat_pdf_file_tokens, ) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict +from litellm.types.utils import CallTypes @dataclass @@ -1614,17 +1618,25 @@ async def count_request_input_tokens( raw_body=raw_body, models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))), ) + chat_pdf_files: Final = ( + inline_pdf_file_count(request_body.get("messages")) + if CallTypes.acompletion in (get_call_types_for_route(route) or ()) + else 0 + ) return MappingProxyType( { model: largest for model, names in counted_models.items() - if (largest := _largest_count(counts, names)) is not None + if (largest := _largest_count(counts, names, chat_pdf_files)) is not None } ) -def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None: - return max((counts[name] for name in names if name in counts), default=None) +def _largest_count(counts: Mapping[str, int], names: Sequence[str], chat_pdf_files: int) -> int | None: + return max( + (counts[name] + openai_chat_pdf_file_tokens(name, chat_pdf_files) for name in names if name in counts), + default=None, + ) def _estimate_input_tokens( diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 6e144723259..da68c5a9b85 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -13,12 +13,14 @@ from __future__ import annotations import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX from litellm.litellm_core_utils.token_counter import ( deployment_openai_model, pdf_page_pricing, @@ -29,7 +31,7 @@ from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction, holds_compaction_summary, ) -from litellm.router_utils.deployment_pricing import deployment_pricing_models +from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -45,6 +47,8 @@ if TYPE_CHECKING: TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 +_OPENAI_PREFIX: Final = "openai/" + _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -142,19 +146,55 @@ def _count_input_tokens_for_models( def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: """The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment - serving it. + routing would serve it from: the ones named `model`, else those of the one wildcard route that serves it. - An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing. A + An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed + `openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A deployment that reads PDF pages and prices them unlike `model` counts as its model minus the provider prefix, the name `/v1/messages/count_tokens` counts with. Any other deployment counts as the requested name with the encoding tiktoken gives that name as written, without loading its vendor's tokenizer. A name no deployment serves counts as written.""" - deployments: Final = (llm_router.get_model_list(model_name=model, team_id=team_id) or ()) if llm_router else () + deployments: Final = _routed_deployments(model, llm_router, team_id) if llm_router else () return tuple(dict.fromkeys(_counted_name(model, deployment) for deployment in deployments)) or (model,) +def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]: + named: Final = ( + *llm_router.get_all_deployments(model_name=model, team_id=team_id), + *llm_router.get_model_list_from_model_alias(model_name=model), + *llm_router.get_model_list_from_routing_groups(model_name=model), + ) + return named or llm_router.wildcard_route_deployments(model, team_id) + + +def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int: + """What OpenAI chat completions bills on top of the counted pages of `pdf_files` inline PDF files, when + `counted_model`, a name from `models_counted_for`, is served by OpenAI itself.""" + return OPENAI_CHAT_PDF_FILE_TOKENS * pdf_files if counted_model.startswith(_OPENAI_PREFIX) else 0 + + +def inline_pdf_file_count(messages: object) -> int: + return sum(1 for part in _content_parts(messages) if _is_inline_pdf_file(part)) + + +def _content_parts(messages: object) -> Iterator[Mapping[str, object]]: + contents: Final = (_as_list(str_keyed(message).get("content")) for message in _as_list(messages)) + return (str_keyed(part) for part in chain.from_iterable(contents)) + + +def _as_list(value: object) -> Sequence[object]: + return value if isinstance(value, list) else () + + +def _is_inline_pdf_file(part: Mapping[str, object]) -> bool: + file_data: Final = str_keyed(part.get("file")).get("file_data") + return part.get("type") == "file" and isinstance(file_data, str) and file_data.startswith(PDF_DATA_URL_PREFIX) + + def _counted_name(model: str, deployment: object) -> str: openai_model: Final = deployment_openai_model(deployment) + if openai_model is not None and _served_by_openai(deployment): + return _OPENAI_PREFIX + openai_model.removeprefix(_OPENAI_PREFIX) if openai_model is not None: return openai_model served: Final = next(iter(deployment_pricing_models(deployment)), model) @@ -163,6 +203,21 @@ def _counted_name(model: str, deployment: object) -> str: return unresolved_tokenizer_model(model) +def _served_by_openai(deployment: object) -> bool: + params: Final = str_keyed(str_keyed(deployment).get("litellm_params")) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return False + try: + _, served_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + except litellm.exceptions.BadRequestError: + return False + return served_provider == "openai" + + def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 0b9b610dc17..54b11f4f851 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -9134,6 +9134,30 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep assert stash.itpm_reserved_tokens == claude_count +def test_project_itpm_adds_what_openai_chat_bills_on_top_of_a_pdf_files_pages_on_chat_completions_only() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}} + for name, model in (("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + ] + ) + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), + counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id), + ) + data: Final = {"messages": _one_page_pdf_message()} + + assert handler._estimate_precise_input_tokens(data, "on-openai", "acompletion") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "acompletion") + OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert handler._estimate_precise_input_tokens(data, "on-openai", "anthropic_messages") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "anthropic_messages") + ) + + def _extraction_request(state: str, call_id: str) -> dict[str, object]: return { "model": "invoice-reader", diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 6c30e2aeac2..a87ac7fa7c0 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -469,6 +469,33 @@ async def _input_tokens(router: Router, model: str) -> dict[str, int]: ) +async def _reserved_input_tokens(router: Router, model: str, route: str, messages: list[dict[str, object]]) -> int: + counts: Final = await count_request_input_tokens( + request_body={"model": model, "messages": messages}, route=route, llm_router=router, team_id=None + ) + return counts[model] + + +@pytest.mark.asyncio +async def test_an_openai_chat_request_reserves_what_openai_chat_bills_on_top_of_each_pdf_files_pages() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = _router_with_groups(("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + two_pdfs: Final = [*_one_page_pdf_message(), *_one_page_pdf_message()] + + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", two_pdfs) + + 2 * OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/messages", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/messages", two_pdfs) + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) + ) + + @pytest.mark.asyncio async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count(): router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol")) diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index ffa70b8e5aa..82f1d021cb8 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -80,12 +80,16 @@ def _router(*deployments: tuple[str, str]) -> Router: ("pdf-reports", "azure/gpt-5.6-sol", "azure/gpt-5.6-sol"), ("pdf-reports", "azure/responses/gpt-5.6-sol", "azure/responses/gpt-5.6-sol"), ("pdf-reports", "openrouter/openai/gpt-5.6-sol", "openrouter/openai/gpt-5.6-sol"), + ("pdf-reports", "openai/gpt-5.6-sol", "openai/gpt-5.6-sol"), + ("pdf-reports", "gpt-5.6-sol", "openai/gpt-5.6-sol"), ], ids=[ "gpt-named-group-served-by-claude", "custom-group-served-by-azure-gpt", "custom-group-served-by-azure-responses-gpt", "custom-group-served-by-openrouter-gpt", + "custom-group-served-by-openai-gpt", + "custom-group-served-by-unprefixed-openai-gpt", ], ) def test_a_model_group_counts_like_the_deployment_that_serves_it( @@ -155,6 +159,23 @@ def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): assert models_counted_for("reports", router, "team-b") == ("reports",) +def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard(): + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "anthropic/*", "api_key": "x"}}, + { + "model_name": "team-a-gpt", + "litellm_params": {"model": "azure/gpt-*", "api_key": "x", "api_base": "https://x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "gpt-*"}, + }, + ] + ) + + assert models_counted_for("gpt-5.6-sol", router, "team-a") == ("azure/gpt-5.6-sol",) + assert models_counted_for("gpt-5.6-sol", router, "team-b") == models_counted_for("gpt-5.6-sol", router, None) + assert "azure/gpt-5.6-sol" not in models_counted_for("gpt-5.6-sol", router, "team-b") + + def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: model: Final = "claude-sonnet-5-5" body_without_tool_choice: Final = {