diff --git a/litellm/__init__.py b/litellm/__init__.py index fe11884a484..b7d46c197f2 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() diff --git a/litellm/constants.py b/litellm/constants.py index 42ad624026c..ef70f1c8163 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -752,7 +752,12 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches" ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 +OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," +PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 +PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000 +OPENAI_MODEL_NAME_MAX_DASHES: Final = 16 +OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { "low": 1, diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f42e4eae08f..69505309ef0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -7,9 +7,11 @@ import re import struct from bisect import bisect_right from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate +from dataclasses import dataclass +from functools import cached_property +from itertools import accumulate, chain, islice from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast import anyio import anyio.lowlevel @@ -32,7 +34,12 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_CHAT_MODES, + OPENAI_MODEL_NAME_MAX_DASHES, + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, + PDF_MAX_FORM_XOBJECT_DEPTH, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, @@ -604,6 +611,35 @@ TokenCounterFunction = Callable[[str], int] Type for a function that counts tokens in a string. """ +PdfPagePricing: TypeAlias = Literal["openai", "claude"] + + +class _PdfBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfContent(Protocol): + def get_data(self) -> bytes: ... + + +class _PdfPage(Protocol): + @property + def mediabox(self) -> _PdfBox: ... + + def extract_text(self) -> str: ... + + def get_contents(self) -> _PdfContent | None: ... + + +class _PdfReader(Protocol): + @property + def pages(self) -> Sequence[_PdfPage]: ... + + EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -660,7 +696,7 @@ class _MessageCountParams: if uses_legacy_message_accounting(model): self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n self.tokens_per_name = -1 # if there's a name, the role is omitted - elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms: + elif _is_openai_chat_model(actual_model): self.tokens_per_message = 3 self.tokens_per_name = 1 else: @@ -669,6 +705,61 @@ class _MessageCountParams: self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) self.tool_section = chat_template_tool_section(model) + self._model = model + + @cached_property + def pdf_pricing(self) -> PdfPagePricing: + return pdf_page_pricing(self._model) + + +def _is_openai_chat_model(model: str) -> bool: + return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms + + +def _openai_deployment_name(model: str) -> str: + name: Final = model.rpartition("/")[2] + return name.split(":")[1] if name.startswith("ft:") else name + + +def _openai_model_family(model: str) -> str | None: + words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES) + prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) + listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) + return None if listed is None or _is_listed_non_chat_model(listed) else listed + + +def _untyped_cost_map() -> object: + return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped + + +def _cost_map_entry(model: str) -> Mapping[str, object]: + cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {} + return _pdf_dictionary(cost_map.get(model)) or {} + + +def _is_azure_openai_chat_model(model: str) -> bool: + name: Final = _openai_deployment_name(model) + entry: Final = _cost_map_entry(f"azure/{name}") + return ( + entry.get("litellm_provider") == "azure" + and entry.get("mode") in OPENAI_CHAT_MODES + and (entry.get("supports_pdf_input") is True or name.startswith("gpt-")) + ) + + +def _is_openai_or_azure_model(model: str) -> bool: + if _is_listed_non_chat_model(_openai_deployment_name(model)): + return False + return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) + + +def pdf_page_pricing(model: str | None) -> PdfPagePricing: + return "openai" if _is_openai_or_azure_model(model or "") else "claude" + + +def reads_pdf_pages(model: str) -> bool: + entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2]) + return entry.get("supports_pdf_input") is True def token_counter( @@ -840,6 +931,7 @@ def _count_messages( value, use_default_image_token_count, default_token_count, + params.pdf_pricing, ) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -984,7 +1076,7 @@ def _get_exact_count_function( return count_tokens if tokenizer_json["type"] == "openai_tokenizer": - encoding: Final = openai_tokenizer_encoding(model) + encoding: Final = openai_tokenizer_encoding(model or "") def encode_length(text: str) -> int: return _encoding_count(encoding, text) @@ -1004,18 +1096,30 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": return get_encoding(openai_tokenizer_encoding_name(model)) +def _tiktoken_model_name(model: str) -> str: + listed_name: Final = _fix_model_name(model) + if listed_name != "gpt-3.5-turbo": + return listed_name + family: Final = _openai_model_family(model) + if family is not None: + return _fix_model_name(family) + return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else listed_name + + +def _is_listed_non_chat_model(model: str) -> bool: + modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}")) + return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes) + + def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose - - model_to_use: Final = _fix_model_name(model) + model_to_use: Final = _tiktoken_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base" def uses_legacy_message_accounting(model: str) -> bool: @@ -1124,7 +1228,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) -def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: +def _pdf_dictionary(node: object) -> Mapping[str, object] | None: + return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key + + +def _pdf_value(container: Mapping[str, object], key: str) -> object: + return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] + + +_PdfXObjects: TypeAlias = Mapping[str, object] | None +_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfContentSummary: + drawn_names: tuple[str, ...] + draws_inline_image: bool + + +_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfWalk: + frontier: tuple[_PdfDrawing, ...] + seen: frozenset[tuple[int, int]] + summaries: Mapping[int, _PdfContentSummary | None] + draws: int + + +def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects: + resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) + if resources is None: + return inherited + return _pdf_dictionary(_pdf_value(resources, "/XObject")) + + +_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " +_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" +_PDF_XOBJECT_DRAW: Final = re.compile( + rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])" +) +_PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: + try: + return raw.decode(charset) + except UnicodeDecodeError: + return None + + +def _pdf_name(escaped: bytes) -> str: + raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped) + decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS) + return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1")) + + +def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary: + data: Final = content.get_data() + draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1) + return _PdfContentSummary( + drawn_names=tuple(_pdf_name(draw[1]) for draw in draws), + draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None, + ) + + +def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None: + from pypdf.generic import StreamObject + + if not isinstance(form, StreamObject): + return None + return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped + + +def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]: + summary, xobjects = drawing + if xobjects is None: + return () + resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names) + return tuple((xobject, xobjects) for xobject in resolved if xobject is not None) + + +def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]: + xobject, inherited = use + return id(xobject), id(_pdf_xobjects(xobject, inherited)) + + +def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None: + draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier) + if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE: + return None + drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + return None + forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"} + unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen) + unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries} + summaries: Final = MappingProxyType( + {**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}} + ) + drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen) + return _PdfWalk( + frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None), + seen=walk.seen | frozenset(forms), + summaries=summaries, + draws=draws, + ) + + +def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: + contents: Final = page.get_contents() + page_dictionary: Final = _pdf_dictionary(page) + if contents is None or page_dictionary is None: + return False + walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned + frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),), + seen=frozenset(), + summaries=MappingProxyType({}), + draws=0, + ) + for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): + if not walk.frontier: + return False + if any(summary.draws_inline_image for summary, _ in walk.frontier): + return True + if (next_walk := _pdf_next_walk(walk)) is None: + return True + walk = next_walk + return bool(walk.frontier) + + +def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + if _pdf_page_draws_raster_image(page): + return _claude_pdf_page_tokens(page, count_function) + text: Final = page.extract_text() or "" + return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 + + +def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens( + float(page.mediabox.width), float(page.mediabox.height) + ) + + +def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int: + match pdf_pricing: + case "openai": + return _openai_pdf_page_tokens(page, count_function) + case "claude": + return _claude_pdf_page_tokens(page, count_function) + + +def _count_inline_pdf_tokens( + data_url: str, + count_function: TokenCounterFunction, + pdf_pricing: PdfPagePricing = "claude", +) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None try: @@ -1133,12 +1395,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed + _PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) ) + return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None @@ -1148,8 +1408,9 @@ def _count_opaque_document_tokens( data_url: str, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: - pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function) + pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing) if pdf_tokens is not None: return pdf_tokens return calculate_img_tokens( @@ -1164,6 +1425,7 @@ def _count_document_tokens( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: source: Final = document["source"] metadata_tokens: Final = sum( @@ -1176,10 +1438,10 @@ def _count_document_tokens( if isinstance(content, str): return metadata_tokens + count_function(content) return metadata_tokens + _count_content_list( - count_function, content, use_default_image_token_count, default_token_count + count_function, content, use_default_image_token_count, default_token_count, pdf_pricing ) return metadata_tokens + _count_opaque_document_tokens( - _anthropic_image_source_data(source), count_function, use_default_image_token_count + _anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing ) @@ -1187,6 +1449,7 @@ def _count_file_tokens( file_value: object, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" if not isinstance(file_value, Mapping): @@ -1196,7 +1459,9 @@ def _count_file_tokens( name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 if not isinstance(file_data, str) or not file_data: return name_tokens - return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count) + return name_tokens + _count_opaque_document_tokens( + file_data, count_function, use_default_image_token_count, pdf_pricing + ) def _count_anthropic_content( @@ -1204,6 +1469,7 @@ def _count_anthropic_content( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """ Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). @@ -1238,6 +1504,7 @@ def _count_anthropic_content( field_value, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif isinstance(field_value, dict): tokens += count_function(str(field_value)) @@ -1264,6 +1531,7 @@ def _count_content_list( ], use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """Recursively count tokens from a list of content blocks.""" try: @@ -1288,12 +1556,14 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] == "file": num_tokens += _count_file_tokens( c.get("file"), count_function, use_default_image_token_count, + pdf_pricing, ) elif c["type"] in ("tool_use", "tool_result", "search_result"): num_tokens += _count_anthropic_content( @@ -1301,6 +1571,7 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] in ("thinking", "redacted_thinking"): # Claude extended thinking content block diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index e44d9304ac1..6a3ae98f9d4 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.proxy.spend_tracking.input_tokens import models_counted_for from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) +def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]: + from litellm.proxy.proxy_server import llm_router + + return models_counted_for(model, llm_router, team_id) + + def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: from litellm.proxy.proxy_server import llm_router @@ -878,6 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, + counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router, media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None @@ -889,6 +897,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._counted_models_resolver = counted_models_resolver self._media_model_resolver = media_model_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver @@ -4073,8 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type=call_type, include_media=False, ) - raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + counted_models: Final = ( + self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,) + ) + raw_estimated_input_tokens: Final = max( + await asyncio.gather( + *( + offload_token_count(self._estimate_precise_input_tokens)( + data=data, model=model, call_type=call_type + ) + for model in counted_models + ) + ) ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..3f3256a059b 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -7,6 +7,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain from types import MappingProxyType from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast @@ -39,7 +40,11 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model +from litellm.proxy.spend_tracking.input_tokens import ( + count_input_tokens, + count_input_tokens_for_model, + models_counted_for, +) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router @@ -270,6 +275,7 @@ async def reserve_budget_for_request( request_body=request_body, route=route, llm_router=llm_router, + team_id=valid_token.team_id, raw_body=raw_body, ) @@ -1591,6 +1597,7 @@ async def count_request_input_tokens( request_body: dict, route: str, llm_router: Router | None, + team_id: str | None, raw_body: bytes | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. @@ -1599,7 +1606,25 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) + counted_models: Final = MappingProxyType( + {model: models_counted_for(model, llm_router, team_id) for model in models} + ) + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))), + ) + return MappingProxyType( + { + model: largest + for model, names in counted_models.items() + if (largest := _largest_count(counts, names)) is not None + } + ) + + +def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None: + return max((counts[name] for name in names if name in counts), default=None) def _estimate_input_tokens( diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 927152296e9..35b0854d693 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -15,10 +15,11 @@ import asyncio import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, @@ -33,6 +34,9 @@ from litellm.rust_bridge.token_counter import ( rust_tokenizer, ) +if TYPE_CHECKING: + from litellm.router import Router + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -130,6 +134,22 @@ def _count_input_tokens_for_models( ) +def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + """The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that + reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix, + the name `/v1/messages/count_tokens` counts with.""" + served: Final = ( + llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else () + ) + return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,) + + +def _counted_name(model: str, served_model: str) -> str: + if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model): + return model + return served_model.partition("/")[2] or served_model + + def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: diff --git a/pyproject.toml b/pyproject.toml index 00cdb1587aa..6acd863701a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -312,6 +312,7 @@ benchmarks = [ "pytest-codspeed==4.3.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", + "pypdf>=6.21.2,<7.0", ] [build-system] diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..b2a86459438 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,126 @@ +"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed. + +Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size.""" + +import base64 +from itertools import accumulate, chain +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import token_counter + +TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." + + +def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str: + escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(), + *xobjects, + ) + header: Final = b"%PDF-1.4\n" + objects: Final = tuple( + f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1) + ) + offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1]))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n" + ).encode() + return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + + +def _form(resources_number: int, drawing: bytes) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _count_pdf(pdf_base64: str) -> int: + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}} + return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}]) + + +def _page_of_drawing_operators() -> str: + return _pdf_base64("", "<< >>", b" q Q" * 50_000) + + +def _page_of_forms_sharing_resources() -> str: + form_count: Final = 300 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count)) + return _pdf_base64( + TEXT_LINE, + "<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>", + b" q 10 0 0 10 0 0 cm /X0 Do Q", + (forms[0], shared_resources, *forms), + ) + + +def _page_of_parents_sharing_a_child_form() -> str: + parent_count: Final = 100 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child)) + + +PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators() +PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources() +PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form() +BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b"")) +TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b"")) +LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000 +NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8 +UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE) + + +@pytest.mark.benchmark +def test_pdf_page_of_drawing_operators_and_no_image() -> None: + assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None: + assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None: + assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_token_counter_long_unknown_model_name() -> None: + assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index b02e0732817..607f8240905 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -29,7 +29,11 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -1878,11 +1882,11 @@ def test_anthropic_image_block_without_source_raises(): ) -def _count_user_content(content: list[dict]) -> int: +def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int: from litellm.litellm_core_utils.token_counter import token_counter return token_counter( - model="anthropic/claude-fable-5", + model=model, messages=[{"role": "user", "content": content}], use_default_image_token_count=True, ) @@ -1978,14 +1982,27 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): ) -def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: +def _pdf_base64( + pages: tuple[str, ...], + width: int = 612, + height: int = 792, + xobjects: tuple[bytes, ...] = (), + drawing: bytes = b"", + page_resources: str | None = None, +) -> str: + """`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every + page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary.""" + first_xobject: Final = 4 + 2 * len(pages) + page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else "" + resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>" + def page_objects(index: int, text: str) -> tuple[bytes, bytes]: escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream" page: Final = ( f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] " - f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>" + f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>" ).encode() return content, page @@ -1995,6 +2012,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", *chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)), + *xobjects, ) header: Final = b"%PDF-1.4\n" objects: Final = tuple( @@ -2073,6 +2091,401 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. assert priced_as_image < priced_by_page +_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." +_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years." +_ONE_PIXEL_IMAGE: Final = ( + b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 " + b"/Length 1 >>\nstream\n\x00\nendstream" +) + + +_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q" +_DRAW_X1: Final = b"/X1 Do" + + +def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _xobject_resources(xobject_number: int) -> bytes: + return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode() + + +def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]: + return {"type": "document", "source": _pdf_source(pdf_base64)} + + +def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]: + file_data: Final = "data:application/pdf;base64," + pdf_base64 + return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}} + + +def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]: + return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]} + + +def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int: + return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model) + + +def _text_cost(model: str, *lines: str) -> int: + return sum( + _count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines + ) + + +def _rendered_page_cost(width: int = 612, height: int = 792) -> int: + return _pdf_cost("my-private-model", _pdf_base64(("",), width, height)) + + +@pytest.mark.parametrize( + "wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"] +) +@pytest.mark.parametrize( + "model", + ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"], +) +def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( + model: str, wrap: Callable[[str], Mapping[str, object]] +): + """OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page + image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed + +28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens + for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini. + + Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF + that Azure gpt-5.6-sol billed at 61. + """ + one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap) + + assert one_page == _text_cost(model, _REVENUE_LINE) + 13 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == ( + _text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13 + ) + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page + assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0 + assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page + + +@pytest.mark.parametrize( + "xobjects", + [(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)], + ids=["image_on_the_page", "image_inside_a_form"], +) +def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page( + xobjects: tuple[bytes, ...], +): + """A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation + never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on + gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn + chart page billed no image. + """ + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost() + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == ( + _rendered_page_cost(300, 300) + ) + + +def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws(): + """Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured + 2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0 + + +def test_openai_model_prices_no_image_for_forms_that_draw_none(): + """A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk.""" + model: Final = "gpt-4o" + self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6)) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + + +_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + + +def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject(): + """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured + 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" + model: Final = "gpt-4o" + form_drawing_inline_image: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode() + + _INLINE_IMAGE + + b"\nendstream" + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == ( + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == ( + _rendered_page_cost() + ) + + +def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk(): + """A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and + priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image.""" + model: Final = "gpt-4o" + depth: Final = 2000 + first: Final = 6 + chain_objects: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth) + ) + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + + +@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"]) +def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes): + """A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two + parents share reaches the image only through the parent whose resources list it.""" + model: Final = "gpt-4o" + child_without_resources: Final = ( + b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + ) + parents_and_child: Final = ( + _form_xobject(7, b"/C Do"), + b"<< /XObject << /C 10 0 R >> >>", + _form_xobject(9, b"/C Do"), + b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>", + child_without_resources, + _ONE_PIXEL_IMAGE, + ) + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=parents_and_child, + drawing=drawing, + page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize( + ("drawing", "draws_image"), + [(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)], + ids=["comment-before-do", "do-inside-a-comment"], +) +def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool): + """A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after + it nor draws anything itself.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == ( + _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13) + ) + + +@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) +def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str): + """A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf + decodes it, so an image drawn under such a name still prices the page as rendered.""" + model: Final = "gpt-4o" + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=(_ONE_PIXEL_IMAGE,), + drawing=f" {escaped_name} Do".encode(), + page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)]) +def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered( + parent_count: int, draws_too_many: bool +): + """A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects + between them, so a page of forms nested to blow up the walk cannot stall the request.""" + model: Final = "gpt-4o" + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2)) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + + assert _pdf_cost( + model, + _pdf_base64( + (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources + ), + ) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13) + + +_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.6-sol", + "gpt-5.4-mini", + "gpt-6-sol", + "azure/gpt-41", + "azure/gpt-5.6-sol", + "eu/gpt-5.6-sol", + "gpt-5.6-sol-2026-07-09", + ], +) +def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): + """OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on + /v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a + non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base.""" + o200k: Final = tiktoken.get_encoding("o200k_base") + + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE)) + + +@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")]) +def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str): + assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"] +) +def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "openrouter/openai/gpt-4o", + "github_copilot/gpt-4o", + "azure_ai/gpt-4o", + "azure/responses/gpt-5.6-sol", + "azure/o_series/o3-mini", + ], +) +def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str): + """The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes + a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on + 2026-10-20, closer to OpenAI's pricing than to Claude's 1534.""" + bare_name: Final = model.rpartition("/")[2] + pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0) + + assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image) + assert _pdf_cost(model, _pdf_base64(("",))) == 0 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "text-embedding-ada-002-v2", + "tts-1", + "whisper-1", + "azure/whisper-1", + "github_copilot/text-embedding-3-small-inference", + "azure_text/gpt-35-turbo-instruct", + "ft:davinci-002:my-org::abc123", + ], +) +def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str): + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + + +@pytest.mark.parametrize( + ("fine_tune", "base_model"), + [ + ("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"), + ("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"), + ], +) +def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str): + pdf: Final = _pdf_base64((_REVENUE_LINE,)) + + assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13 + assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=base_model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", + [ + "", + "anthropic/claude-fable-5", + "DeepSeek-V4-Flash-0731", + "my-private-model", + "azure/mistral-large-latest", + "command-r-plus", + ], +) +def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): + """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps + Claude's text plus rendered page image, 1534 tokens for a blank Letter page.""" + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534 + + +def test_messages_reach_token_count_prices_pdf_pages_like_token_counter(): + from litellm.litellm_core_utils.token_counter import messages_reach_token_count + + messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}] + total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages) + + assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + ( + _text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13 + ) + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False + + def _png_data_url(width: int, height: int) -> str: ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() @@ -2385,6 +2798,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t } +def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None: + sample: Final = "A custom tokenizer passed with no model still counts." + + assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == ( + len(tiktoken.get_encoding("cl100k_base").encode(sample)) + ) + + def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: text_value: Final = "A tokenizer fallback should preserve the model encoding." custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index e056e723a04..c82ecb72b09 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -9099,6 +9099,36 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit( assert "expensive" in str(denied.value.detail) +@pytest.mark.asyncio +async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None: + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), + counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"), + ) + messages: Final = _one_page_pdf_message() + caller: Final = UserAPIKeyAuth( + api_key=hash_token("sk-pdf-itpm"), + project_id="pdf-itpm", + project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=cache, + data={"model": "gpt-4o", "messages": messages, "max_tokens": 10}, + call_type="acompletion", + ) + + stash: Final = get_request_stash() + assert stash is not None + claude_count: Final = handler._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5") + assert claude_count > handler._estimate_precise_input_tokens({"messages": messages}, "gpt-4o") + assert stash.itpm_reserved_tokens == claude_count + + def _extraction_request(state: str, call_id: str) -> dict[str, object]: return { "model": "invoice-reader", diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..65a1969ece6 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -426,3 +426,22 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe ) def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None: assert get_budget_window_start(window) == expected + + +@pytest.mark.asyncio +async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None: + from litellm.proxy.spend_tracking.budget_reservation import count_request_input_tokens + from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()} + router: Final = Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}] + ) + + counts: Final = await count_request_input_tokens( + request_body=request_body, route="/chat/completions", llm_router=router, team_id=None + ) + + assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} + assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o") diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index c7f13651471..148001be32b 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import json from types import MappingProxyType from typing import Final @@ -12,7 +13,9 @@ from litellm.proxy.spend_tracking.input_tokens import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, count_input_tokens, count_input_tokens_for_model, + models_counted_for, ) +from litellm.router import Router CL100K_MODEL: Final = "gpt-4" @@ -30,6 +33,117 @@ async def test_large_input_is_still_counted() -> None: assert isinstance(counts, MappingProxyType) +def _one_page_pdf_message() -> list[dict[str, object]]: + stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET" + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> " + b"/Contents 5 0 R >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + ) + objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1)) + header: Final = b"%PDF-1.4\n" + offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n" + ).encode() + pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + return [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}}, + {"type": "text", "text": "Summarize the report."}, + ], + } + ] + + +def _router(*deployments: tuple[str, str]) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}} + for name, model in deployments + ] + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model"), + [("gpt-4o", "anthropic/claude-haiku-5-5"), ("pdf-reports", "azure/gpt-5.6-sol")], + ids=["gpt-named-group-served-by-claude", "custom-group-served-by-azure-gpt"], +) +def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, deployment_model: str) -> None: + request_body: Final = {"model": group, "messages": _one_page_pdf_message()} + counted_with: Final = deployment_model.partition("/")[2] + + assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,) + assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( + request_body=request_body, model=group + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model"), + [ + ("my-claude", "anthropic/claude-haiku-5-5"), + ("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), + ("my-cohere", "cohere/command-r-plus"), + ("gpt-4o", "cohere/command-r-plus"), + ], + ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"], +) +def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group( + group: str, deployment_model: str +) -> None: + assert models_counted_for(group, _router((group, deployment_model)), None) == (group,) + + +def test_a_custom_named_deployment_counts_with_its_base_model(): + router: Final = Router( + model_list=[ + { + "model_name": "reports", + "litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"}, + "model_info": {"base_model": "azure/gpt-5.6-sol"}, + }, + {"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}}, + ] + ) + + assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",) + assert models_counted_for("drafts", router, None) == ("drafts",) + + +def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): + router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) + + assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol") + assert models_counted_for("not-deployed", router, None) == ("not-deployed",) + assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",) + + +def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): + router: Final = Router( + model_list=[ + {"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}, + { + "model_name": "reports-team-a", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "reports"}, + }, + ] + ) + + assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",) + assert models_counted_for("reports", router, "team-b") == ("reports",) + + def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: model: Final = "claude-sonnet-5-5" body_without_tool_choice: Final = { diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 3513a0ef44c..ba9c0b90498 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -144,9 +144,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "cl100k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), + ("azure/gpt-4o", "o200k_base"), + ("eu/gpt-5.6-sol", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), ("gpt-4.1", "o200k_base"), diff --git a/uv.lock b/uv.lock index b40490112cf..87a86b6812f 100644 --- a/uv.lock +++ b/uv.lock @@ -5118,6 +5118,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -5347,6 +5348,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.21.2,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ]