This commit is contained in:
Mateo Wang 2026-10-29 06:02:44 +00:00 • committed by GitHub
commit d6f220c8fc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1086 additions and 32 deletions

View file

@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
####### COMPLETION MODELS ###################
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_chat_completion_models: Set[str] = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()

View file

@ -752,7 +752,12 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
"low": 1,

View file

@ -7,9 +7,11 @@ import re
import struct
from bisect import bisect_right
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from dataclasses import dataclass
from functools import cached_property
from itertools import accumulate, chain, islice
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
import anyio
import anyio.lowlevel
@ -32,7 +34,12 @@ from litellm.constants import (
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
MAX_TILE_HEIGHT,
MAX_TILE_WIDTH,
OPENAI_CHAT_MODES,
OPENAI_MODEL_NAME_MAX_DASHES,
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
PDF_DATA_URL_PREFIX,
PDF_MAX_FORM_XOBJECT_DEPTH,
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
TOKEN_COUNTER_MAX_EXACT_CHARS,
@ -604,6 +611,35 @@ TokenCounterFunction = Callable[[str], int]
Type for a function that counts tokens in a string.
"""
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
class _PdfBox(Protocol):
@property
def width(self) -> float: ...
@property
def height(self) -> float: ...
class _PdfContent(Protocol):
def get_data(self) -> bytes: ...
class _PdfPage(Protocol):
@property
def mediabox(self) -> _PdfBox: ...
def extract_text(self) -> str: ...
def get_contents(self) -> _PdfContent | None: ...
class _PdfReader(Protocol):
@property
def pages(self) -> Sequence[_PdfPage]: ...
EXTRAPOLATION_SAMPLES: Final = 16
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
@ -660,7 +696,7 @@ class _MessageCountParams:
if uses_legacy_message_accounting(model):
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
self.tokens_per_name = -1 # if there's a name, the role is omitted
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
elif _is_openai_chat_model(actual_model):
self.tokens_per_message = 3
self.tokens_per_name = 1
else:
@ -669,6 +705,61 @@ class _MessageCountParams:
self.tokens_per_name = 1
self.count_function = _get_count_function(model, custom_tokenizer)
self.tool_section = chat_template_tool_section(model)
self._model = model
@cached_property
def pdf_pricing(self) -> PdfPagePricing:
return pdf_page_pricing(self._model)
def _is_openai_chat_model(model: str) -> bool:
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
def _openai_deployment_name(model: str) -> str:
name: Final = model.rpartition("/")[2]
return name.split(":")[1] if name.startswith("ft:") else name
def _openai_model_family(model: str) -> str | None:
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
return None if listed is None or _is_listed_non_chat_model(listed) else listed
def _untyped_cost_map() -> object:
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
def _cost_map_entry(model: str) -> Mapping[str, object]:
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
return _pdf_dictionary(cost_map.get(model)) or {}
def _is_azure_openai_chat_model(model: str) -> bool:
name: Final = _openai_deployment_name(model)
entry: Final = _cost_map_entry(f"azure/{name}")
return (
entry.get("litellm_provider") == "azure"
and entry.get("mode") in OPENAI_CHAT_MODES
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
)
def _is_openai_or_azure_model(model: str) -> bool:
if _is_listed_non_chat_model(_openai_deployment_name(model)):
return False
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
def reads_pdf_pages(model: str) -> bool:
entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2])
return entry.get("supports_pdf_input") is True
def token_counter(
@ -840,6 +931,7 @@ def _count_messages(
value,
use_default_image_token_count,
default_token_count,
params.pdf_pricing,
)
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -984,7 +1076,7 @@ def _get_exact_count_function(
return count_tokens
if tokenizer_json["type"] == "openai_tokenizer":
encoding: Final = openai_tokenizer_encoding(model)
encoding: Final = openai_tokenizer_encoding(model or "")
def encode_length(text: str) -> int:
return _encoding_count(encoding, text)
@ -1004,18 +1096,30 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
return get_encoding(openai_tokenizer_encoding_name(model))
def _tiktoken_model_name(model: str) -> str:
listed_name: Final = _fix_model_name(model)
if listed_name != "gpt-3.5-turbo":
return listed_name
family: Final = _openai_model_family(model)
if family is not None:
return _fix_model_name(family)
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else listed_name
def _is_listed_non_chat_model(model: str) -> bool:
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding."""
from litellm.utils import print_verbose
model_to_use: Final = _fix_model_name(model)
model_to_use: Final = _tiktoken_model_name(model)
if "gpt-4o" in model_to_use:
return "o200k_base"
try:
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return "cl100k_base"
return "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
def uses_legacy_message_accounting(model: str) -> bool:
@ -1124,7 +1228,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
def _pdf_value(container: Mapping[str, object], key: str) -> object:
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
_PdfXObjects: TypeAlias = Mapping[str, object] | None
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfContentSummary:
drawn_names: tuple[str, ...]
draws_inline_image: bool
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfWalk:
frontier: tuple[_PdfDrawing, ...]
seen: frozenset[tuple[int, int]]
summaries: Mapping[int, _PdfContentSummary | None]
draws: int
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
if resources is None:
return inherited
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
_PDF_XOBJECT_DRAW: Final = re.compile(
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
)
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
def _decoded(raw: bytes, charset: str) -> str | None:
try:
return raw.decode(charset)
except UnicodeDecodeError:
return None
def _pdf_name(escaped: bytes) -> str:
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
data: Final = content.get_data()
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
return _PdfContentSummary(
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
)
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
from pypdf.generic import StreamObject
if not isinstance(form, StreamObject):
return None
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
summary, xobjects = drawing
if xobjects is None:
return ()
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
xobject, inherited = use
return id(xobject), id(_pdf_xobjects(xobject, inherited))
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
return None
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
return None
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
summaries: Final = MappingProxyType(
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
)
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
return _PdfWalk(
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
seen=walk.seen | frozenset(forms),
summaries=summaries,
draws=draws,
)
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
contents: Final = page.get_contents()
page_dictionary: Final = _pdf_dictionary(page)
if contents is None or page_dictionary is None:
return False
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
seen=frozenset(),
summaries=MappingProxyType({}),
draws=0,
)
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
if not walk.frontier:
return False
if any(summary.draws_inline_image for summary, _ in walk.frontier):
return True
if (next_walk := _pdf_next_walk(walk)) is None:
return True
walk = next_walk
return bool(walk.frontier)
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
if _pdf_page_draws_raster_image(page):
return _claude_pdf_page_tokens(page, count_function)
text: Final = page.extract_text() or ""
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
float(page.mediabox.width), float(page.mediabox.height)
)
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
match pdf_pricing:
case "openai":
return _openai_pdf_page_tokens(page, count_function)
case "claude":
return _claude_pdf_page_tokens(page, count_function)
def _count_inline_pdf_tokens(
data_url: str,
count_function: TokenCounterFunction,
pdf_pricing: PdfPagePricing = "claude",
) -> int | None:
if not data_url.startswith(PDF_DATA_URL_PREFIX):
return None
try:
@ -1133,12 +1395,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
return None
try:
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
return sum(
count_function(page.extract_text() or "")
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
for page in reader.pages
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
)
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
except Exception as e:
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
return None
@ -1148,8 +1408,9 @@ def _count_opaque_document_tokens(
data_url: str,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
if pdf_tokens is not None:
return pdf_tokens
return calculate_img_tokens(
@ -1164,6 +1425,7 @@ def _count_document_tokens(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
source: Final = document["source"]
metadata_tokens: Final = sum(
@ -1176,10 +1438,10 @@ def _count_document_tokens(
if isinstance(content, str):
return metadata_tokens + count_function(content)
return metadata_tokens + _count_content_list(
count_function, content, use_default_image_token_count, default_token_count
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
)
return metadata_tokens + _count_opaque_document_tokens(
_anthropic_image_source_data(source), count_function, use_default_image_token_count
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
)
@ -1187,6 +1449,7 @@ def _count_file_tokens(
file_value: object,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
if not isinstance(file_value, Mapping):
@ -1196,7 +1459,9 @@ def _count_file_tokens(
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
if not isinstance(file_data, str) or not file_data:
return name_tokens
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
return name_tokens + _count_opaque_document_tokens(
file_data, count_function, use_default_image_token_count, pdf_pricing
)
def _count_anthropic_content(
@ -1204,6 +1469,7 @@ def _count_anthropic_content(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
@ -1238,6 +1504,7 @@ def _count_anthropic_content(
field_value,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif isinstance(field_value, dict):
tokens += count_function(str(field_value))
@ -1264,6 +1531,7 @@ def _count_content_list(
],
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""Recursively count tokens from a list of content blocks."""
try:
@ -1288,12 +1556,14 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] == "file":
num_tokens += _count_file_tokens(
c.get("file"),
count_function,
use_default_image_token_count,
pdf_pricing,
)
elif c["type"] in ("tool_use", "tool_result", "search_result"):
num_tokens += _count_anthropic_content(
@ -1301,6 +1571,7 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block

View file

@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
canonical_provider_batch_id,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.spend_tracking.input_tokens import models_counted_for
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
from litellm.router_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
return resolve_model_group_alias(llm_router.model_group_alias, model)
def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]:
from litellm.proxy.proxy_server import llm_router
return models_counted_for(model, llm_router, team_id)
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
from litellm.proxy.proxy_server import llm_router
@ -878,6 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
time_provider: Callable[[], datetime] | None = None,
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
ptu_team_ceiling_resolver: Callable[
[str, str], PTUTeamCeiling | None
@ -889,6 +897,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._time_provider = time_provider or datetime.now
self._tag_rate_limit_resolver = tag_rate_limit_resolver
self._model_group_resolver = model_group_resolver
self._counted_models_resolver = counted_models_resolver
self._media_model_resolver = media_model_resolver
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
self._fail_closed_resolver = fail_closed_resolver
@ -4073,8 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type=call_type,
include_media=False,
)
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=requested_model, call_type=call_type
counted_models: Final = (
self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,)
)
raw_estimated_input_tokens: Final = max(
await asyncio.gather(
*(
offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=model, call_type=call_type
)
for model in counted_models
)
)
)
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
estimated_output_tokens: Final = (

View file

@ -7,6 +7,7 @@ import time
from collections.abc import AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from itertools import chain
from types import MappingProxyType
from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast
@ -39,7 +40,11 @@ from litellm.proxy.common_utils.user_api_key_cache import (
tag_cache_key,
team_membership_reservation_cache_key,
)
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
from litellm.proxy.spend_tracking.input_tokens import (
count_input_tokens,
count_input_tokens_for_model,
models_counted_for,
)
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -270,6 +275,7 @@ async def reserve_budget_for_request(
request_body=request_body,
route=route,
llm_router=llm_router,
team_id=valid_token.team_id,
raw_body=raw_body,
)
@ -1591,6 +1597,7 @@ async def count_request_input_tokens(
request_body: dict,
route: str,
llm_router: Router | None,
team_id: str | None,
raw_body: bytes | None = None,
) -> Mapping[str, int]:
"""Input-token count per candidate model, counted once per request.
@ -1599,7 +1606,25 @@ async def count_request_input_tokens(
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
if not models:
return MappingProxyType({})
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
counted_models: Final = MappingProxyType(
{model: models_counted_for(model, llm_router, team_id) for model in models}
)
counts: Final = await count_input_tokens(
request_body=request_body,
raw_body=raw_body,
models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))),
)
return MappingProxyType(
{
model: largest
for model, names in counted_models.items()
if (largest := _largest_count(counts, names)) is not None
}
)
def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None:
return max((counts[name] for name in names if name in counts), default=None)
def _estimate_input_tokens(

View file

@ -15,10 +15,11 @@ import asyncio
import json
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages
from litellm.responses.litellm_completion_transformation.compaction_items import (
billable_input_since_newest_compaction,
holds_compaction_summary,
@ -33,6 +34,9 @@ from litellm.rust_bridge.token_counter import (
rust_tokenizer,
)
if TYPE_CHECKING:
from litellm.router import Router
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
@ -130,6 +134,22 @@ def _count_input_tokens_for_models(
)
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
"""The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that
reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix,
the name `/v1/messages/count_tokens` counts with."""
served: Final = (
llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else ()
)
return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,)
def _counted_name(model: str, served_model: str) -> str:
if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model):
return model
return served_model.partition("/")[2] or served_model
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
try:
if "messages" in request_body:

View file

@ -312,6 +312,7 @@ benchmarks = [
"pytest-codspeed==4.3.0",
"mcp>=2.3.0,<3",
"a2a-sdk==1.1.0",
"pypdf>=6.21.2,<7.0",
]
[build-system]

View file

@ -0,0 +1,126 @@
"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed.
Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size."""
import base64
from itertools import accumulate, chain
from typing import Final
import pytest
from litellm.litellm_core_utils.token_counter import token_counter
TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str:
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
bodies: Final = (
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(),
*xobjects,
)
header: Final = b"%PDF-1.4\n"
objects: Final = tuple(
f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1)
)
offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1])))
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
f"{offset:010d} 00000 n \n".encode() for offset in offsets
)
trailer: Final = (
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n"
).encode()
return base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
def _form(resources_number: int, drawing: bytes) -> bytes:
return (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
f"/Length {len(drawing)} >>\nstream\n".encode()
+ drawing
+ b"\nendstream"
)
def _count_pdf(pdf_base64: str) -> int:
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}}
return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}])
def _page_of_drawing_operators() -> str:
return _pdf_base64("", "<< >>", b" q Q" * 50_000)
def _page_of_forms_sharing_resources() -> str:
form_count: Final = 300
shared_resources: Final = (
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
).encode()
forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count))
return _pdf_base64(
TEXT_LINE,
"<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>",
b" q 10 0 0 10 0 0 cm /X0 Do Q",
(forms[0], shared_resources, *forms),
)
def _page_of_parents_sharing_a_child_form() -> str:
parent_count: Final = 100
first: Final = 6
child_number: Final = first + 2 * parent_count
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
child: Final = (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
+ child_drawing
+ b"\nendstream"
)
parents: Final = tuple(
chain.from_iterable(
(_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
for index in range(parent_count)
)
)
page_resources: Final = (
"<< /Font << /F1 3 0 R >> /XObject << "
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
+ " >> >>"
)
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child))
PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators()
PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources()
PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form()
BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b""))
TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b""))
LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000
NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8
UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE)
@pytest.mark.benchmark
def test_pdf_page_of_drawing_operators_and_no_image() -> None:
assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT
@pytest.mark.benchmark
def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None:
assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT
@pytest.mark.benchmark
def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None:
assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT
@pytest.mark.benchmark
def test_token_counter_long_unknown_model_name() -> None:
assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT

View file

@ -29,7 +29,11 @@ import litellm
from litellm import decode, encode, get_modified_max_tokens
from litellm import token_counter as token_counter_old
import litellm.constants
from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
from litellm.constants import (
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.token_counter import (
_encoding_count,
@ -1878,11 +1882,11 @@ def test_anthropic_image_block_without_source_raises():
)
def _count_user_content(content: list[dict]) -> int:
def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int:
from litellm.litellm_core_utils.token_counter import token_counter
return token_counter(
model="anthropic/claude-fable-5",
model=model,
messages=[{"role": "user", "content": content}],
use_default_image_token_count=True,
)
@ -1978,14 +1982,27 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
)
def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str:
def _pdf_base64(
pages: tuple[str, ...],
width: int = 612,
height: int = 792,
xobjects: tuple[bytes, ...] = (),
drawing: bytes = b"",
page_resources: str | None = None,
) -> str:
"""`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every
page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary."""
first_xobject: Final = 4 + 2 * len(pages)
page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else ""
resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>"
def page_objects(index: int, text: str) -> tuple[bytes, bytes]:
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1")
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream"
page: Final = (
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] "
f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>"
f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>"
).encode()
return content, page
@ -1995,6 +2012,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) ->
f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(),
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
*chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)),
*xobjects,
)
header: Final = b"%PDF-1.4\n"
objects: Final = tuple(
@ -2073,6 +2091,401 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest.
assert priced_as_image < priced_by_page
_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years."
_ONE_PIXEL_IMAGE: Final = (
b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 "
b"/Length 1 >>\nstream\n\x00\nendstream"
)
_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q"
_DRAW_X1: Final = b"/X1 Do"
def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes:
return (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
f"/Length {len(drawing)} >>\nstream\n".encode()
+ drawing
+ b"\nendstream"
)
def _xobject_resources(xobject_number: int) -> bytes:
return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode()
def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]:
return {"type": "document", "source": _pdf_source(pdf_base64)}
def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]:
file_data: Final = "data:application/pdf;base64," + pdf_base64
return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}}
def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]:
return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]}
def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int:
return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model)
def _text_cost(model: str, *lines: str) -> int:
return sum(
_count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines
)
def _rendered_page_cost(width: int = 612, height: int = 792) -> int:
return _pdf_cost("my-private-model", _pdf_base64(("",), width, height))
@pytest.mark.parametrize(
"wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"]
)
@pytest.mark.parametrize(
"model",
["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"],
)
def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing(
model: str, wrap: Callable[[str], Mapping[str, object]]
):
"""OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page
image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed
+28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens
for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini.
Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF
that Azure gpt-5.6-sol billed at 61.
"""
one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap)
assert one_page == _text_cost(model, _REVENUE_LINE) + 13
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == (
_text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13
)
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page
assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0
assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page
@pytest.mark.parametrize(
"xobjects",
[(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)],
ids=["image_on_the_page", "image_inside_a_form"],
)
def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page(
xobjects: tuple[bytes, ...],
):
"""A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation
never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on
gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn
chart page billed no image.
"""
model: Final = "gpt-4o"
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == (
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
)
assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost()
assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == (
_rendered_page_cost(300, 300)
)
def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws():
"""Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured
2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images."""
model: Final = "gpt-4o"
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == (
_text_cost(model, _REVENUE_LINE) + 13
)
form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE)
assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0
def test_openai_model_prices_no_image_for_forms_that_draw_none():
"""A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk."""
model: Final = "gpt-4o"
self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6))
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == (
_text_cost(model, _REVENUE_LINE) + 13
)
_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q"
def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject():
"""An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured
2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o."""
model: Final = "gpt-4o"
form_drawing_inline_image: Final = (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode()
+ _INLINE_IMAGE
+ b"\nendstream"
)
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == (
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
)
assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == (
_rendered_page_cost()
)
assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == (
_rendered_page_cost()
)
def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk():
"""A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and
priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image."""
model: Final = "gpt-4o"
depth: Final = 2000
first: Final = 6
chain_objects: Final = tuple(
chain.from_iterable(
(_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth)
)
)
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == (
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
)
@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"])
def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes):
"""A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two
parents share reaches the image only through the parent whose resources list it."""
model: Final = "gpt-4o"
child_without_resources: Final = (
b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream"
)
parents_and_child: Final = (
_form_xobject(7, b"/C Do"),
b"<< /XObject << /C 10 0 R >> >>",
_form_xobject(9, b"/C Do"),
b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>",
child_without_resources,
_ONE_PIXEL_IMAGE,
)
assert _pdf_cost(
model,
_pdf_base64(
("",),
xobjects=parents_and_child,
drawing=drawing,
page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>",
),
) == _rendered_page_cost()
@pytest.mark.parametrize(
("drawing", "draws_image"),
[(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)],
ids=["comment-before-do", "do-inside-a-comment"],
)
def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool):
"""A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after
it nor draws anything itself."""
model: Final = "gpt-4o"
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == (
_text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13)
)
@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"])
def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str):
"""A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf
decodes it, so an image drawn under such a name still prices the page as rendered."""
model: Final = "gpt-4o"
assert _pdf_cost(
model,
_pdf_base64(
("",),
xobjects=(_ONE_PIXEL_IMAGE,),
drawing=f" {escaped_name} Do".encode(),
page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>",
),
) == _rendered_page_cost()
@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)])
def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered(
parent_count: int, draws_too_many: bool
):
"""A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects
between them, so a page of forms nested to blow up the walk cannot stall the request."""
model: Final = "gpt-4o"
first: Final = 6
child_number: Final = first + 2 * parent_count
child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2))
child: Final = (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
+ child_drawing
+ b"\nendstream"
)
parents: Final = tuple(
chain.from_iterable(
(_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
for index in range(parent_count)
)
)
page_resources: Final = (
"<< /Font << /F1 3 0 R >> /XObject << "
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
+ " >> >>"
)
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
assert _pdf_cost(
model,
_pdf_base64(
(_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources
),
) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13)
_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8
@pytest.mark.parametrize(
"model",
[
"gpt-5.6-sol",
"gpt-5.4-mini",
"gpt-6-sol",
"azure/gpt-41",
"azure/gpt-5.6-sol",
"eu/gpt-5.6-sol",
"gpt-5.6-sol-2026-07-09",
],
)
def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str):
"""OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on
/v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a
non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base."""
o200k: Final = tiktoken.get_encoding("o200k_base")
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE))
@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")])
def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str):
assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new(
model=model, text=_NON_LATIN_SAMPLE
)
@pytest.mark.parametrize(
"model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"]
)
def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str):
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE)
)
@pytest.mark.parametrize(
"model",
[
"openrouter/openai/gpt-4o",
"github_copilot/gpt-4o",
"azure_ai/gpt-4o",
"azure/responses/gpt-5.6-sol",
"azure/o_series/o3-mini",
],
)
def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str):
"""The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes
a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on
2026-10-20, closer to OpenAI's pricing than to Claude's 1534."""
bare_name: Final = model.rpartition("/")[2]
pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0)
assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image)
assert _pdf_cost(model, _pdf_base64(("",))) == 0
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE)
)
@pytest.mark.parametrize(
"model",
[
"text-embedding-ada-002-v2",
"tts-1",
"whisper-1",
"azure/whisper-1",
"github_copilot/text-embedding-3-small-inference",
"azure_text/gpt-35-turbo-instruct",
"ft:davinci-002:my-org::abc123",
],
)
def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str):
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE)
)
assert _pdf_cost(model, _pdf_base64(("",))) == 1534
@pytest.mark.parametrize(
("fine_tune", "base_model"),
[
("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"),
("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"),
],
)
def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str):
pdf: Final = _pdf_base64((_REVENUE_LINE,))
assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13
assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new(
model=base_model, text=_NON_LATIN_SAMPLE
)
@pytest.mark.parametrize(
"model",
[
"",
"anthropic/claude-fable-5",
"DeepSeek-V4-Flash-0731",
"my-private-model",
"azure/mistral-large-latest",
"command-r-plus",
],
)
def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str):
"""Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps
Claude's text plus rendered page image, 1534 tokens for a blank Letter page."""
assert _pdf_cost(model, _pdf_base64(("",))) == 1534
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534
def test_messages_reach_token_count_prices_pdf_pages_like_token_counter():
from litellm.litellm_core_utils.token_counter import messages_reach_token_count
messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}]
total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages)
assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + (
_text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13
)
assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True
assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False
def _png_data_url(width: int, height: int) -> str:
ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode()
@ -2385,6 +2798,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t
}
def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None:
sample: Final = "A custom tokenizer passed with no model still counts."
assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == (
len(tiktoken.get_encoding("cl100k_base").encode(sample))
)
def test_empty_custom_tokenizer_uses_model_tokenizer() -> None:
text_value: Final = "A tokenizer fallback should preserve the model encoding."
custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value)

View file

@ -9099,6 +9099,36 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit(
assert "expensive" in str(denied.value.detail)
@pytest.mark.asyncio
async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None:
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
cache: Final = DualCache()
handler: Final = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(cache),
counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"),
)
messages: Final = _one_page_pdf_message()
caller: Final = UserAPIKeyAuth(
api_key=hash_token("sk-pdf-itpm"),
project_id="pdf-itpm",
project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}},
)
await handler.async_pre_call_hook(
user_api_key_dict=caller,
cache=cache,
data={"model": "gpt-4o", "messages": messages, "max_tokens": 10},
call_type="acompletion",
)
stash: Final = get_request_stash()
assert stash is not None
claude_count: Final = handler._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5")
assert claude_count > handler._estimate_precise_input_tokens({"messages": messages}, "gpt-4o")
assert stash.itpm_reserved_tokens == claude_count
def _extraction_request(state: str, call_id: str) -> dict[str, object]:
return {
"model": "invoice-reader",

View file

@ -426,3 +426,22 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
)
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
assert get_budget_window_start(window) == expected
@pytest.mark.asyncio
async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None:
from litellm.proxy.spend_tracking.budget_reservation import count_request_input_tokens
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()}
router: Final = Router(
model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}]
)
counts: Final = await count_request_input_tokens(
request_body=request_body, route="/chat/completions", llm_router=router, team_id=None
)
assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")}
assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o")

View file

@ -2,6 +2,7 @@
from __future__ import annotations
import base64
import json
from types import MappingProxyType
from typing import Final
@ -12,7 +13,9 @@ from litellm.proxy.spend_tracking.input_tokens import (
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS,
count_input_tokens,
count_input_tokens_for_model,
models_counted_for,
)
from litellm.router import Router
CL100K_MODEL: Final = "gpt-4"
@ -30,6 +33,117 @@ async def test_large_input_is_still_counted() -> None:
assert isinstance(counts, MappingProxyType)
def _one_page_pdf_message() -> list[dict[str, object]]:
stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET"
bodies: Final = (
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> "
b"/Contents 5 0 R >>",
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
)
objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1))
header: Final = b"%PDF-1.4\n"
offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects)))
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
f"{offset:010d} 00000 n \n".encode() for offset in offsets
)
trailer: Final = (
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n"
).encode()
pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
return [
{
"role": "user",
"content": [
{"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}},
{"type": "text", "text": "Summarize the report."},
],
}
]
def _router(*deployments: tuple[str, str]) -> Router:
return Router(
model_list=[
{"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}}
for name, model in deployments
]
)
@pytest.mark.parametrize(
("group", "deployment_model"),
[("gpt-4o", "anthropic/claude-haiku-5-5"), ("pdf-reports", "azure/gpt-5.6-sol")],
ids=["gpt-named-group-served-by-claude", "custom-group-served-by-azure-gpt"],
)
def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, deployment_model: str) -> None:
request_body: Final = {"model": group, "messages": _one_page_pdf_message()}
counted_with: Final = deployment_model.partition("/")[2]
assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,)
assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model(
request_body=request_body, model=group
)
@pytest.mark.parametrize(
("group", "deployment_model"),
[
("my-claude", "anthropic/claude-haiku-5-5"),
("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"),
("my-cohere", "cohere/command-r-plus"),
("gpt-4o", "cohere/command-r-plus"),
],
ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"],
)
def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group(
group: str, deployment_model: str
) -> None:
assert models_counted_for(group, _router((group, deployment_model)), None) == (group,)
def test_a_custom_named_deployment_counts_with_its_base_model():
router: Final = Router(
model_list=[
{
"model_name": "reports",
"litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"},
"model_info": {"base_model": "azure/gpt-5.6-sol"},
},
{"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}},
]
)
assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",)
assert models_counted_for("drafts", router, None) == ("drafts",)
def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is():
router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol"))
assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol")
assert models_counted_for("not-deployed", router, None) == ("not-deployed",)
assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",)
def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
router: Final = Router(
model_list=[
{"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}},
{
"model_name": "reports-team-a",
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"},
"model_info": {"team_id": "team-a", "team_public_model_name": "reports"},
},
]
)
assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",)
assert models_counted_for("reports", router, "team-b") == ("reports",)
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None:
model: Final = "claude-sonnet-5-5"
body_without_tool_choice: Final = {

View file

@ -144,9 +144,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None)
("gemini/gemini-2.5-pro", "cl100k_base"),
("mistral/mistral-large-latest", "cl100k_base"),
("my-router-alias", "cl100k_base"),
("azure/gpt-4o", "cl100k_base"),
("command-r-plus", "cl100k_base"),
("gpt-4o", "o200k_base"),
("azure/gpt-4o", "o200k_base"),
("eu/gpt-5.6-sol", "o200k_base"),
("gpt-4o-mini", "o200k_base"),
("gpt-4o-2024-08-06", "o200k_base"),
("gpt-4.1", "o200k_base"),

2
uv.lock generated
View file

@ -5118,6 +5118,7 @@ admin-mcp = [
benchmarks = [
{ name = "a2a-sdk" },
{ name = "mcp" },
{ name = "pypdf" },
{ name = "pytest" },
{ name = "pytest-codspeed" },
]
@ -5347,6 +5348,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1
benchmarks = [
{ name = "a2a-sdk", specifier = "==1.1.0" },
{ name = "mcp", specifier = ">=2.3.0,<3" },
{ name = "pypdf", specifier = ">=6.21.2,<7.0" },
{ name = "pytest", specifier = "==9.0.3" },
{ name = "pytest-codspeed", specifier = "==4.3.0" },
]