fix(proxy): count a team's wildcard route the way routing serves it and reserve OpenAI chat's per-PDF charge

This commit is contained in:
mateo-berri 2026-10-29 10:34:48 -07:00
parent 1ea5aafe7a
commit 20dcfde079
7 changed files with 158 additions and 9 deletions

View file

@ -753,6 +753,7 @@ ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000

View file

@ -76,7 +76,11 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
canonical_provider_batch_id,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.spend_tracking.input_tokens import models_counted_for
from litellm.proxy.spend_tracking.input_tokens import (
inline_pdf_file_count,
models_counted_for,
openai_chat_pdf_file_tokens,
)
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
@ -514,6 +518,7 @@ _TPM_FLOOR_FRACTION: Final = 4
# check on data["input"] must exclude these call types, or a Responses call
# gets misclassified as an embedding and skips output-token reservation/caps.
RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses")
CHAT_COMPLETION_API_CALL_TYPES: Final = (CallTypes.completion.value, CallTypes.acompletion.value)
EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding")
TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion")
RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value)
@ -4016,6 +4021,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
chat_pdf_files: Final = (
inline_pdf_file_count(countable_messages) if call_type in CHAT_COMPLETION_API_CALL_TYPES else 0
)
try:
estimate: Final = max(
@ -4031,6 +4039,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
use_default_image_token_count=True,
)
)
+ openai_chat_pdf_file_tokens(tokenizer_model, chat_pdf_files)
for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",))
),
)

View file

@ -17,6 +17,7 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.proxy._types import (
@ -43,13 +44,16 @@ from litellm.proxy.common_utils.user_api_key_cache import (
from litellm.proxy.spend_tracking.input_tokens import (
count_input_tokens,
count_input_tokens_for_model,
inline_pdf_file_count,
models_counted_for,
openai_chat_pdf_file_tokens,
)
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
from litellm.types.utils import CallTypes
@dataclass
@ -1614,17 +1618,25 @@ async def count_request_input_tokens(
raw_body=raw_body,
models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))),
)
chat_pdf_files: Final = (
inline_pdf_file_count(request_body.get("messages"))
if CallTypes.acompletion in (get_call_types_for_route(route) or ())
else 0
)
return MappingProxyType(
{
model: largest
for model, names in counted_models.items()
if (largest := _largest_count(counts, names)) is not None
if (largest := _largest_count(counts, names, chat_pdf_files)) is not None
}
)
def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None:
return max((counts[name] for name in names if name in counts), default=None)
def _largest_count(counts: Mapping[str, int], names: Sequence[str], chat_pdf_files: int) -> int | None:
return max(
(counts[name] + openai_chat_pdf_file_tokens(name, chat_pdf_files) for name in names if name in counts),
default=None,
)
def _estimate_input_tokens(

View file

@ -13,12 +13,14 @@ from __future__ import annotations
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX
from litellm.litellm_core_utils.token_counter import (
deployment_openai_model,
pdf_page_pricing,
@ -29,7 +31,7 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
billable_input_since_newest_compaction,
holds_compaction_summary,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.token_counter import (
@ -45,6 +47,8 @@ if TYPE_CHECKING:
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
_OPENAI_PREFIX: Final = "openai/"
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
@ -142,19 +146,55 @@ def _count_input_tokens_for_models(
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
"""The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment
serving it.
routing would serve it from: the ones named `model`, else those of the one wildcard route that serves it.
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing. A
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed
`openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A
deployment that reads PDF pages and prices them unlike `model` counts as its model minus the provider prefix, the
name `/v1/messages/count_tokens` counts with. Any other deployment counts as the requested name with the encoding
tiktoken gives that name as written, without loading its vendor's tokenizer. A name no deployment serves counts as
written."""
deployments: Final = (llm_router.get_model_list(model_name=model, team_id=team_id) or ()) if llm_router else ()
deployments: Final = _routed_deployments(model, llm_router, team_id) if llm_router else ()
return tuple(dict.fromkeys(_counted_name(model, deployment) for deployment in deployments)) or (model,)
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
named: Final = (
*llm_router.get_all_deployments(model_name=model, team_id=team_id),
*llm_router.get_model_list_from_model_alias(model_name=model),
*llm_router.get_model_list_from_routing_groups(model_name=model),
)
return named or llm_router.wildcard_route_deployments(model, team_id)
def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int:
"""What OpenAI chat completions bills on top of the counted pages of `pdf_files` inline PDF files, when
`counted_model`, a name from `models_counted_for`, is served by OpenAI itself."""
return OPENAI_CHAT_PDF_FILE_TOKENS * pdf_files if counted_model.startswith(_OPENAI_PREFIX) else 0
def inline_pdf_file_count(messages: object) -> int:
return sum(1 for part in _content_parts(messages) if _is_inline_pdf_file(part))
def _content_parts(messages: object) -> Iterator[Mapping[str, object]]:
contents: Final = (_as_list(str_keyed(message).get("content")) for message in _as_list(messages))
return (str_keyed(part) for part in chain.from_iterable(contents))
def _as_list(value: object) -> Sequence[object]:
return value if isinstance(value, list) else ()
def _is_inline_pdf_file(part: Mapping[str, object]) -> bool:
file_data: Final = str_keyed(part.get("file")).get("file_data")
return part.get("type") == "file" and isinstance(file_data, str) and file_data.startswith(PDF_DATA_URL_PREFIX)
def _counted_name(model: str, deployment: object) -> str:
openai_model: Final = deployment_openai_model(deployment)
if openai_model is not None and _served_by_openai(deployment):
return _OPENAI_PREFIX + openai_model.removeprefix(_OPENAI_PREFIX)
if openai_model is not None:
return openai_model
served: Final = next(iter(deployment_pricing_models(deployment)), model)
@ -163,6 +203,21 @@ def _counted_name(model: str, deployment: object) -> str:
return unresolved_tokenizer_model(model)
def _served_by_openai(deployment: object) -> bool:
params: Final = str_keyed(str_keyed(deployment).get("litellm_params"))
model: Final = params.get("model")
provider: Final = params.get("custom_llm_provider")
if not isinstance(model, str):
return False
try:
_, served_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=provider if isinstance(provider, str) else None
)
except litellm.exceptions.BadRequestError:
return False
return served_provider == "openai"
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
try:
if "messages" in request_body:

View file

@ -9134,6 +9134,30 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep
assert stash.itpm_reserved_tokens == claude_count
def test_project_itpm_adds_what_openai_chat_bills_on_top_of_a_pdf_files_pages_on_chat_completions_only() -> None:
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
router: Final = Router(
model_list=[
{"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}}
for name, model in (("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol"))
]
)
handler: Final = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(DualCache()),
counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id),
)
data: Final = {"messages": _one_page_pdf_message()}
assert handler._estimate_precise_input_tokens(data, "on-openai", "acompletion") == (
handler._estimate_precise_input_tokens(data, "on-azure", "acompletion") + OPENAI_CHAT_PDF_FILE_TOKENS
)
assert handler._estimate_precise_input_tokens(data, "on-openai", "anthropic_messages") == (
handler._estimate_precise_input_tokens(data, "on-azure", "anthropic_messages")
)
def _extraction_request(state: str, call_id: str) -> dict[str, object]:
return {
"model": "invoice-reader",

View file

@ -469,6 +469,33 @@ async def _input_tokens(router: Router, model: str) -> dict[str, int]:
)
async def _reserved_input_tokens(router: Router, model: str, route: str, messages: list[dict[str, object]]) -> int:
counts: Final = await count_request_input_tokens(
request_body={"model": model, "messages": messages}, route=route, llm_router=router, team_id=None
)
return counts[model]
@pytest.mark.asyncio
async def test_an_openai_chat_request_reserves_what_openai_chat_bills_on_top_of_each_pdf_files_pages() -> None:
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
router: Final = _router_with_groups(("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol"))
two_pdfs: Final = [*_one_page_pdf_message(), *_one_page_pdf_message()]
assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", two_pdfs) == (
await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", two_pdfs)
+ 2 * OPENAI_CHAT_PDF_FILE_TOKENS
)
assert await _reserved_input_tokens(router, "on-openai", "/v1/messages", two_pdfs) == (
await _reserved_input_tokens(router, "on-azure", "/v1/messages", two_pdfs)
)
assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) == (
await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", _MULTILINGUAL_MESSAGES)
)
@pytest.mark.asyncio
async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count():
router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol"))

View file

@ -80,12 +80,16 @@ def _router(*deployments: tuple[str, str]) -> Router:
("pdf-reports", "azure/gpt-5.6-sol", "azure/gpt-5.6-sol"),
("pdf-reports", "azure/responses/gpt-5.6-sol", "azure/responses/gpt-5.6-sol"),
("pdf-reports", "openrouter/openai/gpt-5.6-sol", "openrouter/openai/gpt-5.6-sol"),
("pdf-reports", "openai/gpt-5.6-sol", "openai/gpt-5.6-sol"),
("pdf-reports", "gpt-5.6-sol", "openai/gpt-5.6-sol"),
],
ids=[
"gpt-named-group-served-by-claude",
"custom-group-served-by-azure-gpt",
"custom-group-served-by-azure-responses-gpt",
"custom-group-served-by-openrouter-gpt",
"custom-group-served-by-openai-gpt",
"custom-group-served-by-unprefixed-openai-gpt",
],
)
def test_a_model_group_counts_like_the_deployment_that_serves_it(
@ -155,6 +159,23 @@ def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
assert models_counted_for("reports", router, "team-b") == ("reports",)
def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard():
router: Final = Router(
model_list=[
{"model_name": "*", "litellm_params": {"model": "anthropic/*", "api_key": "x"}},
{
"model_name": "team-a-gpt",
"litellm_params": {"model": "azure/gpt-*", "api_key": "x", "api_base": "https://x"},
"model_info": {"team_id": "team-a", "team_public_model_name": "gpt-*"},
},
]
)
assert models_counted_for("gpt-5.6-sol", router, "team-a") == ("azure/gpt-5.6-sol",)
assert models_counted_for("gpt-5.6-sol", router, "team-b") == models_counted_for("gpt-5.6-sol", router, None)
assert "azure/gpt-5.6-sol" not in models_counted_for("gpt-5.6-sol", router, "team-b")
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None:
model: Final = "claude-sonnet-5-5"
body_without_tool_choice: Final = {