From 678c9e487a10d5cab03a21e7b38560ddf949cb6e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:23:16 -0700 Subject: [PATCH 01/11] test(token_counter): price PDF pages for OpenAI and Azure models the way they bill --- .../litellm_core_utils/test_token_counter.py | 183 +++++++++++++++++- 1 file changed, 178 insertions(+), 5 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 309cbd33131..b6a1e25be22 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1684,11 +1684,11 @@ def test_anthropic_image_block_without_source_raises(): ) -def _count_user_content(content: list[dict]) -> int: +def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int: from litellm.litellm_core_utils.token_counter import token_counter return token_counter( - model="anthropic/claude-fable-5", + model=model, messages=[{"role": "user", "content": content}], use_default_image_token_count=True, ) @@ -1784,14 +1784,25 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): ) -def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: +def _pdf_base64( + pages: tuple[str, ...], + width: int = 612, + height: int = 792, + xobjects: tuple[bytes, ...] = (), + drawing: bytes = b"", +) -> str: + """`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every + page's content stream ends with `drawing`.""" + first_xobject: Final = 4 + 2 * len(pages) + page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else "" + def page_objects(index: int, text: str) -> tuple[bytes, bytes]: escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream" page: Final = ( f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] " - f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>" + f"/Resources << /Font << /F1 3 0 R >>{page_xobjects} >> /Contents {4 + 2 * index} 0 R >>" ).encode() return content, page @@ -1801,6 +1812,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", *chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)), + *xobjects, ) header: Final = b"%PDF-1.4\n" objects: Final = tuple( @@ -1879,6 +1891,167 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. assert priced_as_image < priced_by_page +_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." +_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years." +_ONE_PIXEL_IMAGE: Final = ( + b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 " + b"/Length 1 >>\nstream\n\x00\nendstream" +) + + +def _form_xobject(resources_number: int) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + "/Length 0 >>\nstream\n\nendstream" + ).encode() + + +def _xobject_resources(xobject_number: int) -> bytes: + return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode() + + +def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]: + return {"type": "document", "source": _pdf_source(pdf_base64)} + + +def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]: + return {"type": "file", "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + pdf_base64}} + + +def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]: + return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]} + + +def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int: + return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model) + + +def _text_cost(model: str, *lines: str) -> int: + return sum( + _count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines + ) + + +@pytest.mark.parametrize("wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"]) +@pytest.mark.parametrize( + "model", ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"] +) +def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( + model: str, wrap: Callable[[str], Mapping[str, object]] +): + """OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page + image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed + +28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens + for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini. + + Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF + that Azure gpt-5.6-sol billed at 61. + """ + one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap) + + assert one_page == _text_cost(model, _REVENUE_LINE) + 13 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == ( + _text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13 + ) + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page + assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0 + assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page + + +@pytest.mark.parametrize( + "xobjects", + [(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)], + ids=["image_on_the_page", "image_inside_a_form"], +) +def test_openai_model_prices_a_page_that_draws_a_raster_image_as_a_high_detail_page_image(xobjects: tuple[bytes, ...]): + """A page embedding a raster image also bills a picture of the page. It is priced as a high-detail image of the + page's size with gpt-4o's tile math (85 tokens plus 170 per 512 px tile after fitting the short side to 768 px): + https://developers.openai.com/api/docs/guides/images-vision, read 2026-10-20, when a full-page scanned Letter page + billed +798 on gpt-4o and a vector-drawn chart page billed no image. + """ + model: Final = "gpt-4o" + letter_page_image: Final = 85 + 4 * 170 + small_page_image: Final = 85 + 170 + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + letter_page_image + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects)) == letter_page_image + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects)) == small_page_image + + +def test_openai_model_prices_no_image_for_forms_that_draw_none(): + """A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk.""" + model: Final = "gpt-4o" + self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6)) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + + +def test_openai_model_prices_a_page_that_draws_an_inline_image_as_a_high_detail_page_image(): + """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured + 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" + model: Final = "gpt-4o" + inline_image: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=inline_image)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 + ) + + +def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk(): + """A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and + priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image.""" + model: Final = "gpt-4o" + depth: Final = 2000 + first: Final = 6 + chain_objects: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth) + ) + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 + ) + + +_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 + + +@pytest.mark.parametrize("model", ["gpt-5.6-sol", "gpt-5.4-mini", "gpt-6-sol", "azure/gpt-41"]) +def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): + """OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on + /v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a + non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base.""" + o200k: Final = tiktoken.get_encoding("o200k_base") + + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE)) + + +@pytest.mark.parametrize("model", ["", "anthropic/claude-fable-5", "DeepSeek-V4-Flash-0731", "my-private-model"]) +def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): + """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps + Claude's text plus rendered page image, 1534 tokens for a blank Letter page.""" + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534 + + +def test_messages_reach_token_count_prices_pdf_pages_like_token_counter(): + from litellm.litellm_core_utils.token_counter import messages_reach_token_count + + messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}] + total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages) + + assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + ( + _text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13 + ) + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False + + def _png_data_url(width: int, height: int) -> str: ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() From 64ac38cf27a33525fff6924f6f43cfd757b2d22a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:33:39 -0700 Subject: [PATCH 02/11] fix(token_counter): price PDF pages for OpenAI and Azure models the way they bill --- litellm/constants.py | 2 + litellm/litellm_core_utils/token_counter.py | 179 ++++++++++++++++++-- 2 files changed, 164 insertions(+), 17 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index bc94760b187..fd3180ccf76 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -740,7 +740,9 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches" ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 +OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," +PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { "low": 1, diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8aded69aed2..238b239dbdd 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,8 +5,9 @@ import io import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, cast +from functools import partial +from itertools import accumulate, chain +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast import anyio import anyio.lowlevel @@ -29,7 +30,9 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, + PDF_MAX_FORM_XOBJECT_DEPTH, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, @@ -347,6 +350,29 @@ TokenCounterFunction = Callable[[str], int] Type for a function that counts tokens in a string. """ +PdfPagePricing: TypeAlias = Literal["openai", "claude"] + + +class _PdfBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfPage(Protocol): + @property + def mediabox(self) -> _PdfBox: ... + + def extract_text(self, *, visitor_operand_before: Callable[..., None] | None = None) -> str: ... + + +class _PdfReader(Protocol): + @property + def pages(self) -> Sequence[_PdfPage]: ... + + EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -392,6 +418,8 @@ class _MessageCountParams: A class to hold the parameters for counting tokens in messages. """ + pdf_pricing: PdfPagePricing + def __init__( self, model: str, @@ -403,7 +431,7 @@ class _MessageCountParams: if uses_legacy_message_accounting(model): self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n self.tokens_per_name = -1 # if there's a name, the role is omitted - elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms: + elif _is_openai_chat_model(actual_model): self.tokens_per_message = 3 self.tokens_per_name = 1 else: @@ -412,6 +440,30 @@ class _MessageCountParams: self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) self.tool_section = chat_template_tool_section(model) + self.pdf_pricing = _pdf_page_pricing(model) + + +def _is_openai_chat_model(model: str) -> bool: + return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms + + +_OPENAI_FILE_INPUT_PROVIDERS: Final = frozenset({"openai", "azure"}) + + +def _served_through_openai_file_inputs(bare_name: str) -> bool: + entries: Final = (litellm.model_cost.get(key) for key in (bare_name, f"azure/{bare_name}", f"openai/{bare_name}")) + return any( + isinstance(entry, Mapping) and entry.get("litellm_provider") in _OPENAI_FILE_INPUT_PROVIDERS + for entry in entries + ) + + +def _pdf_page_pricing(model: str | None) -> PdfPagePricing: + name: Final = model or "" + bare_name: Final = name.removeprefix("openai/").removeprefix("azure/") + if _is_openai_chat_model(name) or _is_openai_chat_model(bare_name) or _served_through_openai_file_inputs(bare_name): + return "openai" + return "claude" def token_counter( @@ -583,6 +635,7 @@ def _count_messages( value, use_default_image_token_count, default_token_count, + params.pdf_pricing, ) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -743,16 +796,13 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding.""" - from litellm.utils import print_verbose - model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - print_verbose("Warning: model not found. Using cl100k_base encoding.") - return "cl100k_base" + return "o200k_base" def uses_legacy_message_accounting(model: str) -> bool: @@ -860,7 +910,93 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) -def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: +def _pdf_dictionary(node: object) -> Mapping[str, object] | None: + return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key + + +def _pdf_value(container: Mapping[str, object], key: str) -> object: + return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] + + +def _pdf_xobjects(container: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) + xobjects: Final = _pdf_dictionary(_pdf_value(resources, "/XObject")) if resources is not None else None + if xobjects is None: + return () + resolved: Final = (_pdf_dictionary(xobjects[name]) for name in xobjects) + return tuple(xobject for xobject in resolved if xobject is not None) + + +def _pdf_unseen_forms( + xobjects: tuple[Mapping[str, object], ...], seen: frozenset[int] +) -> tuple[Mapping[str, object], ...]: + forms: Final = {id(xobject): xobject for xobject in xobjects if _pdf_value(xobject, "/Subtype") == "/Form"} + return tuple(form for form_id, form in forms.items() if form_id not in seen) + + +def _pdf_resources_draw_raster_image(page: Mapping[str, object]) -> bool: + frontier: tuple[Mapping[str, object], ...] = (page,) # rebind-ok: breadth-first walk, recursion is banned + seen: frozenset[int] = frozenset() # rebind-ok: breadth-first walk, recursion is banned + for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): + if not frontier: + return False + xobjects = tuple(chain.from_iterable(map(_pdf_xobjects, frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject in xobjects): + return True + seen = seen | frozenset(map(id, frontier)) + frontier = _pdf_unseen_forms(xobjects, seen) + return bool(frontier) + + +def _record_inline_image( + inline_images: list[bytes], # mutable-ok: pypdf reports content operators only to a callback + operator: bytes, + *_: object, +) -> None: + if operator == b"INLINE IMAGE": + inline_images.append(operator) + + +def _openai_page_image_tokens(width: float, height: float) -> int: + if width <= 0 or height <= 0: + return 0 + resized_width, resized_height = resize_image_high_res(math.ceil(width), math.ceil(height)) + base_tokens: Final = 85 + return base_tokens + 2 * base_tokens * calculate_tiles_needed(resized_width, resized_height) + + +def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + inline_images: Final[list[bytes]] = [] # mutable-ok: pypdf reports content operators only to a callback + text: Final = page.extract_text(visitor_operand_before=partial(_record_inline_image, inline_images)) or "" + text_tokens: Final = count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 + page_dictionary: Final = _pdf_dictionary(page) + draws_image: Final = bool(inline_images) or ( + page_dictionary is not None and _pdf_resources_draw_raster_image(page_dictionary) + ) + if not draws_image: + return text_tokens + return text_tokens + _openai_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) + + +def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens( + float(page.mediabox.width), float(page.mediabox.height) + ) + + +def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int: + match pdf_pricing: + case "openai": + return _openai_pdf_page_tokens(page, count_function) + case "claude": + return _claude_pdf_page_tokens(page, count_function) + + +def _count_inline_pdf_tokens( + data_url: str, + count_function: TokenCounterFunction, + pdf_pricing: PdfPagePricing = "claude", +) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None try: @@ -869,12 +1005,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed + _PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) ) + return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None @@ -884,8 +1018,9 @@ def _count_opaque_document_tokens( data_url: str, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: - pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function) + pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing) if pdf_tokens is not None: return pdf_tokens return calculate_img_tokens( @@ -900,6 +1035,7 @@ def _count_document_tokens( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: source: Final = document["source"] metadata_tokens: Final = sum( @@ -912,10 +1048,10 @@ def _count_document_tokens( if isinstance(content, str): return metadata_tokens + count_function(content) return metadata_tokens + _count_content_list( - count_function, content, use_default_image_token_count, default_token_count + count_function, content, use_default_image_token_count, default_token_count, pdf_pricing ) return metadata_tokens + _count_opaque_document_tokens( - _anthropic_image_source_data(source), count_function, use_default_image_token_count + _anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing ) @@ -923,6 +1059,7 @@ def _count_file_tokens( file_value: object, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" if not isinstance(file_value, Mapping): @@ -932,7 +1069,9 @@ def _count_file_tokens( name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 if not isinstance(file_data, str) or not file_data: return name_tokens - return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count) + return name_tokens + _count_opaque_document_tokens( + file_data, count_function, use_default_image_token_count, pdf_pricing + ) def _count_anthropic_content( @@ -940,6 +1079,7 @@ def _count_anthropic_content( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """ Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). @@ -974,6 +1114,7 @@ def _count_anthropic_content( field_value, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif isinstance(field_value, dict): tokens += count_function(str(field_value)) @@ -997,6 +1138,7 @@ def _count_content_list( ], use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """Recursively count tokens from a list of content blocks.""" try: @@ -1021,12 +1163,14 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] == "file": num_tokens += _count_file_tokens( c.get("file"), count_function, use_default_image_token_count, + pdf_pricing, ) elif c["type"] in ("tool_use", "tool_result"): num_tokens += _count_anthropic_content( @@ -1034,6 +1178,7 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] in ("thinking", "redacted_thinking"): # Claude extended thinking content block From be041227aca3c504b11213ef9cdb02f154e94d0e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:05:42 -0700 Subject: [PATCH 03/11] fix(token_counter): count azure, data-zone and dated GPT names with their family's encoding --- litellm/litellm_core_utils/token_counter.py | 29 ++++++++++--------- .../litellm_core_utils/test_token_counter.py | 25 ++++++++++++++-- tests/unit/rust_bridge/test_token_counter.py | 3 +- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 238b239dbdd..846a189e581 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -447,23 +447,19 @@ def _is_openai_chat_model(model: str) -> bool: return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms -_OPENAI_FILE_INPUT_PROVIDERS: Final = frozenset({"openai", "azure"}) +def _openai_model_family(model: str) -> str | None: + deployment_name: Final = model.removeprefix("openai/").removeprefix("azure/").rsplit("/", 1)[-1] + words: Final = deployment_name.split("-") + prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) + return next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) -def _served_through_openai_file_inputs(bare_name: str) -> bool: - entries: Final = (litellm.model_cost.get(key) for key in (bare_name, f"azure/{bare_name}", f"openai/{bare_name}")) - return any( - isinstance(entry, Mapping) and entry.get("litellm_provider") in _OPENAI_FILE_INPUT_PROVIDERS - for entry in entries - ) +def _is_openai_or_azure_model(model: str) -> bool: + return _is_openai_chat_model(model) or _openai_model_family(model) is not None def _pdf_page_pricing(model: str | None) -> PdfPagePricing: - name: Final = model or "" - bare_name: Final = name.removeprefix("openai/").removeprefix("azure/") - if _is_openai_chat_model(name) or _is_openai_chat_model(bare_name) or _served_through_openai_file_inputs(bare_name): - return "openai" - return "claude" + return "openai" if _is_openai_or_azure_model(model or "") else "claude" def token_counter( @@ -794,9 +790,16 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": return get_encoding(openai_tokenizer_encoding_name(model)) +def _tiktoken_model_name(model: str) -> str: + listed_name: Final = _fix_model_name(model) + if listed_name != "gpt-3.5-turbo": + return listed_name + return _fix_model_name(_openai_model_family(model) or model) + + def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding.""" - model_to_use: Final = _fix_model_name(model) + model_to_use: Final = _tiktoken_model_name(model) if "gpt-4o" in model_to_use: return "o200k_base" try: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index b6a1e25be22..6544b1243f0 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2021,7 +2021,18 @@ def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk _NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 -@pytest.mark.parametrize("model", ["gpt-5.6-sol", "gpt-5.4-mini", "gpt-6-sol", "azure/gpt-41"]) +@pytest.mark.parametrize( + "model", + [ + "gpt-5.6-sol", + "gpt-5.4-mini", + "gpt-6-sol", + "azure/gpt-41", + "azure/gpt-5.6-sol", + "eu/gpt-5.6-sol", + "gpt-5.6-sol-2026-07-09", + ], +) def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): """OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on /v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a @@ -2031,7 +2042,17 @@ def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE)) -@pytest.mark.parametrize("model", ["", "anthropic/claude-fable-5", "DeepSeek-V4-Flash-0731", "my-private-model"]) +@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")]) +def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str): + assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", + ["", "anthropic/claude-fable-5", "DeepSeek-V4-Flash-0731", "my-private-model", "azure/mistral-large-latest", "command-r-plus"], +) def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps Claude's text plus rendered page image, 1534 tokens for a blank Letter page.""" diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 3af7127a9b3..67eaadf58f9 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -143,9 +143,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "cl100k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), + ("azure/gpt-4o", "o200k_base"), + ("eu/gpt-5.6-sol", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), ("gpt-4.1", "o200k_base"), From 8f0bb86b95e27694607c0b730f768354045848e6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:05:46 -0700 Subject: [PATCH 04/11] fix(token_counter): price only drawn PDF images and keep other gateways' GPT pricing Detect raster images from the page content stream, including inline images on pages with empty resources, and walk each form once. Strip only openai/, azure/ and Azure data-zone prefixes, bound the model family scan, read Azure-only chat models from the cost map, and keep cl100k_base for listed non-chat OpenAI models. --- litellm/__init__.py | 2 +- litellm/constants.py | 2 + litellm/litellm_core_utils/token_counter.py | 136 +++++++++++++----- .../litellm_core_utils/test_token_counter.py | 105 ++++++++++++-- 4 files changed, 195 insertions(+), 50 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0fe3d091c2a..60b003d782d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -589,7 +589,7 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() diff --git a/litellm/constants.py b/litellm/constants.py index fd3180ccf76..39225d47700 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -743,6 +743,8 @@ ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 +AZURE_DATA_ZONES: Final = frozenset({"global", "global-standard", "us", "us-gov", "eu", "apac"}) +OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { "low": 1, diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 846a189e581..1b6a59813e1 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,7 +5,6 @@ import io import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from functools import partial from itertools import accumulate, chain from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast @@ -22,6 +21,7 @@ from litellm.constants import ( ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX, ANTHROPIC_IMAGE_MAX_PIXELS, ANTHROPIC_IMAGE_PIXELS_PER_TOKEN, + AZURE_DATA_ZONES, DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_TOKEN_COUNT, DEFAULT_IMAGE_WIDTH, @@ -30,6 +30,7 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_CHAT_MODES, OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, PDF_MAX_FORM_XOBJECT_DEPTH, @@ -361,11 +362,18 @@ class _PdfBox(Protocol): def height(self) -> float: ... +class _PdfContent(Protocol): + @property + def operations(self) -> Sequence[tuple[object, bytes]]: ... + + class _PdfPage(Protocol): @property def mediabox(self) -> _PdfBox: ... - def extract_text(self, *, visitor_operand_before: Callable[..., None] | None = None) -> str: ... + def extract_text(self) -> str: ... + + def get_contents(self) -> _PdfContent | None: ... class _PdfReader(Protocol): @@ -447,15 +455,38 @@ def _is_openai_chat_model(model: str) -> bool: return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms +def _openai_deployment_name(model: str) -> str: + provider_free: Final = model.removeprefix("openai/").removeprefix("azure/") + zone, _, deployment_name = provider_free.partition("/") + return deployment_name if zone in AZURE_DATA_ZONES else provider_free + + def _openai_model_family(model: str) -> str | None: - deployment_name: Final = model.removeprefix("openai/").removeprefix("azure/").rsplit("/", 1)[-1] - words: Final = deployment_name.split("-") + longest_listed_name: Final = max( + map(len, chain(litellm.open_ai_chat_completion_models, litellm.azure_llms)), default=0 + ) + deployment_name: Final = _openai_deployment_name(model) + words: Final = deployment_name[: longest_listed_name + 1].split("-") prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) return next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) +def _cost_map_entry(model: str) -> Mapping[str, object]: + entry: Final[object] = litellm.model_cost.get(model) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportUnknownArgumentType] # cost map is loaded untyped + return _pdf_dictionary(entry) or {} + + +def _is_azure_openai_chat_model(model: str) -> bool: + entry: Final = _cost_map_entry(f"azure/{_openai_deployment_name(model)}") + return ( + entry.get("litellm_provider") == "azure" + and entry.get("mode") in OPENAI_CHAT_MODES + and entry.get("supports_pdf_input") is True + ) + + def _is_openai_or_azure_model(model: str) -> bool: - return _is_openai_chat_model(model) or _openai_model_family(model) is not None + return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) def _pdf_page_pricing(model: str | None) -> PdfPagePricing: @@ -797,6 +828,11 @@ def _tiktoken_model_name(model: str) -> str: return _fix_model_name(_openai_model_family(model) or model) +def _is_listed_non_chat_model(model: str) -> bool: + mode: Final = _cost_map_entry(model).get("mode") + return mode is not None and mode not in OPENAI_CHAT_MODES + + def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding.""" model_to_use: Final = _tiktoken_model_name(model) @@ -805,7 +841,7 @@ def openai_tokenizer_encoding_name(model: str) -> str: try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - return "o200k_base" + return "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base" def uses_legacy_message_accounting(model: str) -> bool: @@ -921,45 +957,76 @@ def _pdf_value(container: Mapping[str, object], key: str) -> object: return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] -def _pdf_xobjects(container: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: - resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) +_PdfDrawing: TypeAlias = tuple[_PdfContent, Mapping[str, object] | None] + + +def _pdf_resources( + container: Mapping[str, object], inherited: Mapping[str, object] | None +) -> Mapping[str, object] | None: + return _pdf_dictionary(_pdf_value(container, "/Resources")) or inherited + + +def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...]: + content, resources = drawing xobjects: Final = _pdf_dictionary(_pdf_value(resources, "/XObject")) if resources is not None else None if xobjects is None: return () - resolved: Final = (_pdf_dictionary(xobjects[name]) for name in xobjects) - return tuple(xobject for xobject in resolved if xobject is not None) + operand_lists: Final = (operands for operands, operator in content.operations if operator == b"Do") + names: Final[Iterable[object]] = ( + cast(Sequence[object], operands)[0] # cast-ok: pypdf parses an operator's operands into a list + for operands in operand_lists + if isinstance(operands, list) and operands + ) + resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in names if isinstance(name, str)) + return tuple((xobject, resources) for xobject in resolved if xobject is not None) -def _pdf_unseen_forms( - xobjects: tuple[Mapping[str, object], ...], seen: frozenset[int] -) -> tuple[Mapping[str, object], ...]: - forms: Final = {id(xobject): xobject for xobject in xobjects if _pdf_value(xobject, "/Subtype") == "/Form"} - return tuple(form for form_id, form in forms.items() if form_id not in seen) +def _draws_inline_image(content: _PdfContent) -> bool: + return any(operator == b"INLINE IMAGE" for _, operator in content.operations) -def _pdf_resources_draw_raster_image(page: Mapping[str, object]) -> bool: - frontier: tuple[Mapping[str, object], ...] = (page,) # rebind-ok: breadth-first walk, recursion is banned +def _pdf_form_drawing(form: Mapping[str, object], inherited: Mapping[str, object] | None) -> _PdfDrawing | None: + from pypdf.generic import ContentStream, StreamObject + + if not isinstance(form, StreamObject): + return None + return cast(_PdfContent, ContentStream(form, None)), _pdf_resources(form, inherited) # cast-ok: pypdf is untyped + + +def _pdf_unseen_form_drawings( + drawn: tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...], seen: frozenset[int] +) -> tuple[_PdfDrawing, ...]: + forms: Final = {id(xobject): (xobject, inherited) for xobject, inherited in drawn if id(xobject) not in seen} + form_drawings: Final = ( + _pdf_form_drawing(form, inherited) + for form, inherited in forms.values() + if _pdf_value(form, "/Subtype") == "/Form" + ) + return tuple(drawing for drawing in form_drawings if drawing is not None) + + +def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: + contents: Final = page.get_contents() + page_dictionary: Final = _pdf_dictionary(page) + if contents is None or page_dictionary is None: + return False + frontier: tuple[_PdfDrawing, ...] = ( # rebind-ok: breadth-first walk, recursion is banned + (contents, _pdf_resources(page_dictionary, None)), + ) seen: frozenset[int] = frozenset() # rebind-ok: breadth-first walk, recursion is banned for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): if not frontier: return False - xobjects = tuple(chain.from_iterable(map(_pdf_xobjects, frontier))) - if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject in xobjects): + if any(_draws_inline_image(content) for content, _ in frontier): return True - seen = seen | frozenset(map(id, frontier)) - frontier = _pdf_unseen_forms(xobjects, seen) + drawn = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + return True + frontier = _pdf_unseen_form_drawings(drawn, seen) + seen = seen | frozenset(id(xobject) for xobject, _ in drawn) return bool(frontier) -def _record_inline_image( - inline_images: list[bytes], # mutable-ok: pypdf reports content operators only to a callback - operator: bytes, - *_: object, -) -> None: - if operator == b"INLINE IMAGE": - inline_images.append(operator) - - def _openai_page_image_tokens(width: float, height: float) -> int: if width <= 0 or height <= 0: return 0 @@ -969,14 +1036,9 @@ def _openai_page_image_tokens(width: float, height: float) -> int: def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - inline_images: Final[list[bytes]] = [] # mutable-ok: pypdf reports content operators only to a callback - text: Final = page.extract_text(visitor_operand_before=partial(_record_inline_image, inline_images)) or "" + text: Final = page.extract_text() or "" text_tokens: Final = count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 - page_dictionary: Final = _pdf_dictionary(page) - draws_image: Final = bool(inline_images) or ( - page_dictionary is not None and _pdf_resources_draw_raster_image(page_dictionary) - ) - if not draws_image: + if not _pdf_page_draws_raster_image(page): return text_tokens return text_tokens + _openai_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 6544b1243f0..3f428c52700 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1790,11 +1790,13 @@ def _pdf_base64( height: int = 792, xobjects: tuple[bytes, ...] = (), drawing: bytes = b"", + page_resources: str | None = None, ) -> str: """`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every - page's content stream ends with `drawing`.""" + page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary.""" first_xobject: Final = 4 + 2 * len(pages) page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else "" + resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>" def page_objects(index: int, text: str) -> tuple[bytes, bytes]: escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") @@ -1802,7 +1804,7 @@ def _pdf_base64( content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream" page: Final = ( f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] " - f"/Resources << /Font << /F1 3 0 R >>{page_xobjects} >> /Contents {4 + 2 * index} 0 R >>" + f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>" ).encode() return content, page @@ -1899,11 +1901,17 @@ _ONE_PIXEL_IMAGE: Final = ( ) -def _form_xobject(resources_number: int) -> bytes: +_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q" +_DRAW_X1: Final = b"/X1 Do" + + +def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes: return ( f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " - "/Length 0 >>\nstream\n\nendstream" - ).encode() + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) def _xobject_resources(xobject_number: int) -> bytes: @@ -1973,11 +1981,23 @@ def test_openai_model_prices_a_page_that_draws_a_raster_image_as_a_high_detail_p letter_page_image: Final = 85 + 4 * 170 small_page_image: Final = 85 + 170 - assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects)) == ( + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == ( _text_cost(model, _REVENUE_LINE) + 13 + letter_page_image ) - assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects)) == letter_page_image - assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects)) == small_page_image + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == letter_page_image + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == small_page_image + + +def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws(): + """Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured + 2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0 def test_openai_model_prices_no_image_for_forms_that_draw_none(): @@ -1985,20 +2005,31 @@ def test_openai_model_prices_no_image_for_forms_that_draw_none(): model: Final = "gpt-4o" self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6)) - assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form)) == ( + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == ( _text_cost(model, _REVENUE_LINE) + 13 ) +_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + + def test_openai_model_prices_a_page_that_draws_an_inline_image_as_a_high_detail_page_image(): """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" model: Final = "gpt-4o" - inline_image: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + form_drawing_inline_image: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode() + + _INLINE_IMAGE + + b"\nendstream" + ) - assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=inline_image)) == ( + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == ( _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 ) + assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == 85 + 4 * 170 + assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == ( + 85 + 4 * 170 + ) def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk(): @@ -2013,11 +2044,27 @@ def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk ) ) - assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects)) == ( + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == ( _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 ) +@pytest.mark.timeout(10) +def test_openai_model_reads_resources_shared_by_many_forms_once(): + """3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once.""" + model: Final = "gpt-4o" + form_count: Final = 3000 + shared_resources_number: Final = 7 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form_xobject(shared_resources_number, b"/F0 Do") for _ in range(form_count)) + + assert _pdf_cost( + model, _pdf_base64((_REVENUE_LINE,), xobjects=(forms[0], shared_resources, *forms), drawing=_DRAW_X0) + ) == (_text_cost(model, _REVENUE_LINE) + 13) + + _NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 @@ -2049,6 +2096,40 @@ def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deploym ) +@pytest.mark.parametrize("model", ["azure/gpt-chat-latest", "azure/codex-mini"]) +def test_azure_openai_models_only_azure_lists_get_openai_pdf_pricing(model: str): + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 + + +@pytest.mark.parametrize( + "model", ["openrouter/openai/gpt-4o", "github_copilot/gpt-4o", "replicate/openai/gpt-5", "azure_ai/gpt-4o"] +) +def test_gpt_names_on_other_gateways_keep_their_pdf_pricing_and_encoding(model: str): + """Only openai/, azure/ and Azure data zones are stripped, so a GPT name served through another gateway keeps + the PDF pricing and text encoding it had before.""" + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize("model", ["text-embedding-ada-002-v2", "tts-1", "whisper-1"]) +def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k(model: str): + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.timeout(10) +def test_a_very_long_unknown_model_name_counts_promptly(): + """A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried.""" + long_name: Final = "unknown-" + "a-" * 200_000 + + assert token_counter_new(model=long_name, text=_NON_LATIN_SAMPLE) == token_counter_new( + model="my-private-model", text=_NON_LATIN_SAMPLE + ) + + @pytest.mark.parametrize( "model", ["", "anthropic/claude-fable-5", "DeepSeek-V4-Flash-0731", "my-private-model", "azure/mistral-large-latest", "command-r-plus"], From 8e7130b066a2b212a6c972842faff63313e2884e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:10:26 -0700 Subject: [PATCH 05/11] fix(token_counter): price GPT names behind any prefix like the bare name and walk shared forms per resources Prices a PDF by the model name's last path segment, the same name the proxy counts with, keeps listed non-chat models on Claude pricing, walks each form once per inherited resources dictionary so a shared child form drawing an image is not missed, and resolves PDF pricing only when a message carries a PDF --- litellm/constants.py | 2 +- litellm/litellm_core_utils/token_counter.py | 61 ++++++++++------- .../litellm_core_utils/test_token_counter.py | 67 +++++++++++++++---- 3 files changed, 91 insertions(+), 39 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 39225d47700..78bb3973c37 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -743,7 +743,7 @@ ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 -AZURE_DATA_ZONES: Final = frozenset({"global", "global-standard", "us", "us-gov", "eu", "apac"}) +OPENAI_MODEL_NAME_MAX_DASHES: Final = 16 OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 1b6a59813e1..a2333e52253 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,6 +5,7 @@ import io import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from functools import cached_property from itertools import accumulate, chain from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast @@ -21,7 +22,6 @@ from litellm.constants import ( ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX, ANTHROPIC_IMAGE_MAX_PIXELS, ANTHROPIC_IMAGE_PIXELS_PER_TOKEN, - AZURE_DATA_ZONES, DEFAULT_IMAGE_HEIGHT, DEFAULT_IMAGE_TOKEN_COUNT, DEFAULT_IMAGE_WIDTH, @@ -31,6 +31,7 @@ from litellm.constants import ( MAX_TILE_HEIGHT, MAX_TILE_WIDTH, OPENAI_CHAT_MODES, + OPENAI_MODEL_NAME_MAX_DASHES, OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, PDF_MAX_FORM_XOBJECT_DEPTH, @@ -426,8 +427,6 @@ class _MessageCountParams: A class to hold the parameters for counting tokens in messages. """ - pdf_pricing: PdfPagePricing - def __init__( self, model: str, @@ -448,7 +447,11 @@ class _MessageCountParams: self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) self.tool_section = chat_template_tool_section(model) - self.pdf_pricing = _pdf_page_pricing(model) + self._model = model + + @cached_property + def pdf_pricing(self) -> PdfPagePricing: + return _pdf_page_pricing(self._model) def _is_openai_chat_model(model: str) -> bool: @@ -456,17 +459,11 @@ def _is_openai_chat_model(model: str) -> bool: def _openai_deployment_name(model: str) -> str: - provider_free: Final = model.removeprefix("openai/").removeprefix("azure/") - zone, _, deployment_name = provider_free.partition("/") - return deployment_name if zone in AZURE_DATA_ZONES else provider_free + return model.rpartition("/")[2] def _openai_model_family(model: str) -> str | None: - longest_listed_name: Final = max( - map(len, chain(litellm.open_ai_chat_completion_models, litellm.azure_llms)), default=0 - ) - deployment_name: Final = _openai_deployment_name(model) - words: Final = deployment_name[: longest_listed_name + 1].split("-") + words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES) prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) return next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) @@ -486,6 +483,8 @@ def _is_azure_openai_chat_model(model: str) -> bool: def _is_openai_or_azure_model(model: str) -> bool: + if _is_listed_non_chat_model(_openai_deployment_name(model)): + return False return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) @@ -993,18 +992,34 @@ def _pdf_form_drawing(form: Mapping[str, object], inherited: Mapping[str, object return cast(_PdfContent, ContentStream(form, None)), _pdf_resources(form, inherited) # cast-ok: pypdf is untyped +def _pdf_drawing_key(xobject: Mapping[str, object], inherited: Mapping[str, object] | None) -> tuple[int, int]: + return id(xobject), id(_pdf_resources(xobject, inherited)) + + def _pdf_unseen_form_drawings( - drawn: tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...], seen: frozenset[int] + drawn: tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...], seen: frozenset[tuple[int, int]] ) -> tuple[_PdfDrawing, ...]: - forms: Final = {id(xobject): (xobject, inherited) for xobject, inherited in drawn if id(xobject) not in seen} - form_drawings: Final = ( - _pdf_form_drawing(form, inherited) - for form, inherited in forms.values() - if _pdf_value(form, "/Subtype") == "/Form" - ) + forms: Final = { + _pdf_drawing_key(xobject, inherited): (xobject, inherited) + for xobject, inherited in drawn + if _pdf_value(xobject, "/Subtype") == "/Form" + } + form_drawings: Final = (_pdf_form_drawing(*forms[key]) for key in forms.keys() - seen) return tuple(drawing for drawing in form_drawings if drawing is not None) +_PdfWalkStep: TypeAlias = tuple[tuple[_PdfDrawing, ...], frozenset[tuple[int, int]]] + + +def _pdf_walk_step(frontier: tuple[_PdfDrawing, ...], seen: frozenset[tuple[int, int]]) -> _PdfWalkStep | None: + drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + return None + return _pdf_unseen_form_drawings(drawn, seen), seen | frozenset( + _pdf_drawing_key(xobject, inherited) for xobject, inherited in drawn + ) + + def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: contents: Final = page.get_contents() page_dictionary: Final = _pdf_dictionary(page) @@ -1013,17 +1028,15 @@ def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: frontier: tuple[_PdfDrawing, ...] = ( # rebind-ok: breadth-first walk, recursion is banned (contents, _pdf_resources(page_dictionary, None)), ) - seen: frozenset[int] = frozenset() # rebind-ok: breadth-first walk, recursion is banned + seen: frozenset[tuple[int, int]] = frozenset() # rebind-ok: breadth-first walk, recursion is banned for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): if not frontier: return False if any(_draws_inline_image(content) for content, _ in frontier): return True - drawn = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, frontier))) - if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + if (step := _pdf_walk_step(frontier, seen)) is None: return True - frontier = _pdf_unseen_form_drawings(drawn, seen) - seen = seen | frozenset(id(xobject) for xobject, _ in drawn) + frontier, seen = step return bool(frontier) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 3f428c52700..dc687db292a 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2049,6 +2049,32 @@ def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk ) +@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"]) +def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes): + """A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two + parents share reaches the image only through the parent whose resources list it.""" + model: Final = "gpt-4o" + child_without_resources: Final = b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + parents_and_child: Final = ( + _form_xobject(7, b"/C Do"), + b"<< /XObject << /C 10 0 R >> >>", + _form_xobject(9, b"/C Do"), + b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>", + child_without_resources, + _ONE_PIXEL_IMAGE, + ) + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=parents_and_child, + drawing=drawing, + page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>", + ), + ) == 85 + 4 * 170 + + @pytest.mark.timeout(10) def test_openai_model_reads_resources_shared_by_many_forms_once(): """3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once.""" @@ -2102,22 +2128,35 @@ def test_azure_openai_models_only_azure_lists_get_openai_pdf_pricing(model: str) @pytest.mark.parametrize( - "model", ["openrouter/openai/gpt-4o", "github_copilot/gpt-4o", "replicate/openai/gpt-5", "azure_ai/gpt-4o"] + "model", + [ + "openrouter/openai/gpt-4o", + "github_copilot/gpt-4o", + "azure_ai/gpt-4o", + "azure/responses/gpt-5.6-sol", + "azure/o_series/o3-mini", + ], ) -def test_gpt_names_on_other_gateways_keep_their_pdf_pricing_and_encoding(model: str): - """Only openai/, azure/ and Azure data zones are stripped, so a GPT name served through another gateway keeps - the PDF pricing and text encoding it had before.""" +def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str): + """The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes + a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 and a + full-page image at 297 on 2026-10-20, closer to OpenAI's pricing than to Claude's 1534 for either.""" + bare_name: Final = model.rpartition("/")[2] + pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0) + + assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image) + assert _pdf_cost(model, _pdf_base64(("",))) == 0 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize("model", ["text-embedding-ada-002-v2", "tts-1", "whisper-1", "azure/whisper-1"]) +def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str): + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) assert _pdf_cost(model, _pdf_base64(("",))) == 1534 - assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( - tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) - ) - - -@pytest.mark.parametrize("model", ["text-embedding-ada-002-v2", "tts-1", "whisper-1"]) -def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k(model: str): - assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( - tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) - ) @pytest.mark.timeout(10) From 856d155e0f5494f8f3fd69567a2b59675cabc4f1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:55:16 -0700 Subject: [PATCH 06/11] fix(token_counter): keep the rendered-page count for image pages, parse each form once, and match chat models only A PDF page that draws a raster image keeps the rendered-page estimate, since newer GPT models bill such a page near it. The image walk parses each form once and stops at a fixed number of drawn XObjects, so a form shared under many resources dictionaries no longer slows the count. The family prefix match skips non-chat entries, and ft: fine-tunes count like their base model --- litellm/constants.py | 1 + litellm/litellm_core_utils/token_counter.py | 151 ++++++++++-------- .../litellm_core_utils/test_token_counter.py | 132 ++++++++++++--- 3 files changed, 196 insertions(+), 88 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 78bb3973c37..0479347d109 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -743,6 +743,7 @@ ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 +PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000 OPENAI_MODEL_NAME_MAX_DASHES: Final = 16 OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index a2333e52253..801582e1c98 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -5,8 +5,10 @@ import io import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass from functools import cached_property from itertools import accumulate, chain +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast import anyio @@ -35,6 +37,7 @@ from litellm.constants import ( OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, PDF_MAX_FORM_XOBJECT_DEPTH, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, @@ -459,18 +462,24 @@ def _is_openai_chat_model(model: str) -> bool: def _openai_deployment_name(model: str) -> str: - return model.rpartition("/")[2] + name: Final = model.rpartition("/")[2] + return name.split(":")[1] if name.startswith("ft:") else name def _openai_model_family(model: str) -> str | None: words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES) prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) - return next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) + listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) + return None if listed is None or _is_listed_non_chat_model(listed) else listed + + +def _untyped_cost_map() -> object: + return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped def _cost_map_entry(model: str) -> Mapping[str, object]: - entry: Final[object] = litellm.model_cost.get(model) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportUnknownArgumentType] # cost map is loaded untyped - return _pdf_dictionary(entry) or {} + cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {} + return _pdf_dictionary(cost_map.get(model)) or {} def _is_azure_openai_chat_model(model: str) -> bool: @@ -828,8 +837,8 @@ def _tiktoken_model_name(model: str) -> str: def _is_listed_non_chat_model(model: str) -> bool: - mode: Final = _cost_map_entry(model).get("mode") - return mode is not None and mode not in OPENAI_CHAT_MODES + modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}")) + return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes) def openai_tokenizer_encoding_name(model: str) -> str: @@ -956,67 +965,88 @@ def _pdf_value(container: Mapping[str, object], key: str) -> object: return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] -_PdfDrawing: TypeAlias = tuple[_PdfContent, Mapping[str, object] | None] +_PdfXObjects: TypeAlias = Mapping[str, object] | None +_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects] -def _pdf_resources( - container: Mapping[str, object], inherited: Mapping[str, object] | None -) -> Mapping[str, object] | None: - return _pdf_dictionary(_pdf_value(container, "/Resources")) or inherited +@dataclass(frozen=True, slots=True) +class _PdfContentSummary: + drawn_names: tuple[str, ...] + draws_inline_image: bool -def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...]: - content, resources = drawing - xobjects: Final = _pdf_dictionary(_pdf_value(resources, "/XObject")) if resources is not None else None - if xobjects is None: - return () - operand_lists: Final = (operands for operands, operator in content.operations if operator == b"Do") +_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfWalk: + frontier: tuple[_PdfDrawing, ...] + seen: frozenset[tuple[int, int]] + summaries: Mapping[int, _PdfContentSummary | None] + draws: int + + +def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects: + resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) + if resources is None: + return inherited + return _pdf_dictionary(_pdf_value(resources, "/XObject")) + + +def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary: + operations: Final = content.operations + operand_lists: Final = (operands for operands, operator in operations if operator == b"Do") names: Final[Iterable[object]] = ( cast(Sequence[object], operands)[0] # cast-ok: pypdf parses an operator's operands into a list for operands in operand_lists if isinstance(operands, list) and operands ) - resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in names if isinstance(name, str)) - return tuple((xobject, resources) for xobject in resolved if xobject is not None) + return _PdfContentSummary( + drawn_names=tuple(name for name in names if isinstance(name, str)), + draws_inline_image=any(operator == b"INLINE IMAGE" for _, operator in operations), + ) -def _draws_inline_image(content: _PdfContent) -> bool: - return any(operator == b"INLINE IMAGE" for _, operator in content.operations) - - -def _pdf_form_drawing(form: Mapping[str, object], inherited: Mapping[str, object] | None) -> _PdfDrawing | None: +def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None: from pypdf.generic import ContentStream, StreamObject if not isinstance(form, StreamObject): return None - return cast(_PdfContent, ContentStream(form, None)), _pdf_resources(form, inherited) # cast-ok: pypdf is untyped + return _pdf_content_summary(cast(_PdfContent, ContentStream(form, None))) # cast-ok: pypdf is untyped -def _pdf_drawing_key(xobject: Mapping[str, object], inherited: Mapping[str, object] | None) -> tuple[int, int]: - return id(xobject), id(_pdf_resources(xobject, inherited)) +def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]: + summary, xobjects = drawing + if xobjects is None: + return () + resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names) + return tuple((xobject, xobjects) for xobject in resolved if xobject is not None) -def _pdf_unseen_form_drawings( - drawn: tuple[tuple[Mapping[str, object], Mapping[str, object] | None], ...], seen: frozenset[tuple[int, int]] -) -> tuple[_PdfDrawing, ...]: - forms: Final = { - _pdf_drawing_key(xobject, inherited): (xobject, inherited) - for xobject, inherited in drawn - if _pdf_value(xobject, "/Subtype") == "/Form" - } - form_drawings: Final = (_pdf_form_drawing(*forms[key]) for key in forms.keys() - seen) - return tuple(drawing for drawing in form_drawings if drawing is not None) +def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]: + xobject, inherited = use + return id(xobject), id(_pdf_xobjects(xobject, inherited)) -_PdfWalkStep: TypeAlias = tuple[tuple[_PdfDrawing, ...], frozenset[tuple[int, int]]] - - -def _pdf_walk_step(frontier: tuple[_PdfDrawing, ...], seen: frozenset[tuple[int, int]]) -> _PdfWalkStep | None: - drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, frontier))) +def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None: + draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier) + if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE: + return None + drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier))) if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): return None - return _pdf_unseen_form_drawings(drawn, seen), seen | frozenset( - _pdf_drawing_key(xobject, inherited) for xobject, inherited in drawn + forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"} + unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen) + unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries} + summaries: Final = MappingProxyType( + {**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}} + ) + drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen) + return _PdfWalk( + frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None), + seen=walk.seen | frozenset(forms), + summaries=summaries, + draws=draws, ) @@ -1025,35 +1055,28 @@ def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: page_dictionary: Final = _pdf_dictionary(page) if contents is None or page_dictionary is None: return False - frontier: tuple[_PdfDrawing, ...] = ( # rebind-ok: breadth-first walk, recursion is banned - (contents, _pdf_resources(page_dictionary, None)), + walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned + frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),), + seen=frozenset(), + summaries=MappingProxyType({}), + draws=0, ) - seen: frozenset[tuple[int, int]] = frozenset() # rebind-ok: breadth-first walk, recursion is banned for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): - if not frontier: + if not walk.frontier: return False - if any(_draws_inline_image(content) for content, _ in frontier): + if any(summary.draws_inline_image for summary, _ in walk.frontier): return True - if (step := _pdf_walk_step(frontier, seen)) is None: + if (next_walk := _pdf_next_walk(walk)) is None: return True - frontier, seen = step - return bool(frontier) - - -def _openai_page_image_tokens(width: float, height: float) -> int: - if width <= 0 or height <= 0: - return 0 - resized_width, resized_height = resize_image_high_res(math.ceil(width), math.ceil(height)) - base_tokens: Final = 85 - return base_tokens + 2 * base_tokens * calculate_tiles_needed(resized_width, resized_height) + walk = next_walk + return bool(walk.frontier) def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + if _pdf_page_draws_raster_image(page): + return _claude_pdf_page_tokens(page, count_function) text: Final = page.extract_text() or "" - text_tokens: Final = count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 - if not _pdf_page_draws_raster_image(page): - return text_tokens - return text_tokens + _openai_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) + return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index dc687db292a..9114cc5d0b7 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1923,7 +1923,8 @@ def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]: def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]: - return {"type": "file", "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + pdf_base64}} + file_data: Final = "data:application/pdf;base64," + pdf_base64 + return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}} def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]: @@ -1940,9 +1941,16 @@ def _text_cost(model: str, *lines: str) -> int: ) -@pytest.mark.parametrize("wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"]) +def _rendered_page_cost(width: int = 612, height: int = 792) -> int: + return _pdf_cost("my-private-model", _pdf_base64(("",), width, height)) + + @pytest.mark.parametrize( - "model", ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"] + "wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"] +) +@pytest.mark.parametrize( + "model", + ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"], ) def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( model: str, wrap: Callable[[str], Mapping[str, object]] @@ -1971,21 +1979,23 @@ def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( [(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)], ids=["image_on_the_page", "image_inside_a_form"], ) -def test_openai_model_prices_a_page_that_draws_a_raster_image_as_a_high_detail_page_image(xobjects: tuple[bytes, ...]): - """A page embedding a raster image also bills a picture of the page. It is priced as a high-detail image of the - page's size with gpt-4o's tile math (85 tokens plus 170 per 512 px tile after fitting the short side to 768 px): - https://developers.openai.com/api/docs/guides/images-vision, read 2026-10-20, when a full-page scanned Letter page - billed +798 on gpt-4o and a vector-drawn chart page billed no image. +def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page( + xobjects: tuple[bytes, ...], +): + """A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation + never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on + gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn + chart page billed no image. """ model: Final = "gpt-4o" - letter_page_image: Final = 85 + 4 * 170 - small_page_image: Final = 85 + 170 assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == ( - _text_cost(model, _REVENUE_LINE) + 13 + letter_page_image + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost() + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == ( + _rendered_page_cost(300, 300) ) - assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == letter_page_image - assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == small_page_image def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws(): @@ -2013,7 +2023,7 @@ def test_openai_model_prices_no_image_for_forms_that_draw_none(): _INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" -def test_openai_model_prices_a_page_that_draws_an_inline_image_as_a_high_detail_page_image(): +def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject(): """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" model: Final = "gpt-4o" @@ -2024,11 +2034,13 @@ def test_openai_model_prices_a_page_that_draws_an_inline_image_as_a_high_detail_ ) assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == ( - _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == ( + _rendered_page_cost() ) - assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == 85 + 4 * 170 assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == ( - 85 + 4 * 170 + _rendered_page_cost() ) @@ -2045,7 +2057,7 @@ def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk ) assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == ( - _text_cost(model, _REVENUE_LINE) + 13 + 85 + 4 * 170 + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() ) @@ -2054,7 +2066,9 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_ """A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two parents share reaches the image only through the parent whose resources list it.""" model: Final = "gpt-4o" - child_without_resources: Final = b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + child_without_resources: Final = ( + b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + ) parents_and_child: Final = ( _form_xobject(7, b"/C Do"), b"<< /XObject << /C 10 0 R >> >>", @@ -2072,7 +2086,7 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_ drawing=drawing, page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>", ), - ) == 85 + 4 * 170 + ) == _rendered_page_cost() @pytest.mark.timeout(10) @@ -2091,6 +2105,42 @@ def test_openai_model_reads_resources_shared_by_many_forms_once(): ) == (_text_cost(model, _REVENUE_LINE) + 13) +@pytest.mark.timeout(10) +def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly(): + """1,000 forms that each list their own /Resources and draw one shared 4,000-operator child took about 45 + seconds to price, since the child was parsed and walked again under every parent. A page now prices as if it + drew an image once its forms draw more than a fixed number of XObjects, and each form is parsed once.""" + model: Final = "gpt-4o" + parent_count: Final = 1000 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + + assert _pdf_cost( + model, + _pdf_base64( + (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources + ), + ) == _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + + _NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 @@ -2139,8 +2189,8 @@ def test_azure_openai_models_only_azure_lists_get_openai_pdf_pricing(model: str) ) def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str): """The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes - a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 and a - full-page image at 297 on 2026-10-20, closer to OpenAI's pricing than to Claude's 1534 for either.""" + a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on + 2026-10-20, closer to OpenAI's pricing than to Claude's 1534.""" bare_name: Final = model.rpartition("/")[2] pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0) @@ -2151,7 +2201,18 @@ def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name( ) -@pytest.mark.parametrize("model", ["text-embedding-ada-002-v2", "tts-1", "whisper-1", "azure/whisper-1"]) +@pytest.mark.parametrize( + "model", + [ + "text-embedding-ada-002-v2", + "tts-1", + "whisper-1", + "azure/whisper-1", + "github_copilot/text-embedding-3-small-inference", + "azure_text/gpt-35-turbo-instruct", + "ft:davinci-002:my-org::abc123", + ], +) def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str): assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) @@ -2159,6 +2220,22 @@ def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_cl assert _pdf_cost(model, _pdf_base64(("",))) == 1534 +@pytest.mark.parametrize( + ("fine_tune", "base_model"), + [ + ("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"), + ("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"), + ], +) +def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str): + pdf: Final = _pdf_base64((_REVENUE_LINE,)) + + assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13 + assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=base_model, text=_NON_LATIN_SAMPLE + ) + + @pytest.mark.timeout(10) def test_a_very_long_unknown_model_name_counts_promptly(): """A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried.""" @@ -2171,7 +2248,14 @@ def test_a_very_long_unknown_model_name_counts_promptly(): @pytest.mark.parametrize( "model", - ["", "anthropic/claude-fable-5", "DeepSeek-V4-Flash-0731", "my-private-model", "azure/mistral-large-latest", "command-r-plus"], + [ + "", + "anthropic/claude-fable-5", + "DeepSeek-V4-Flash-0731", + "my-private-model", + "azure/mistral-large-latest", + "command-r-plus", + ], ) def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps From da97117dcae9db30360ffbc8a0cf1b5037f22b93 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:31:20 -0700 Subject: [PATCH 07/11] fix(token_counter): count Azure-only chat models with o200k like other OpenAI chat models --- litellm/litellm_core_utils/token_counter.py | 5 ++++- tests/unit/litellm_core_utils/test_token_counter.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 801582e1c98..7546565b2f3 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -833,7 +833,10 @@ def _tiktoken_model_name(model: str) -> str: listed_name: Final = _fix_model_name(model) if listed_name != "gpt-3.5-turbo": return listed_name - return _fix_model_name(_openai_model_family(model) or model) + family: Final = _openai_model_family(model) + if family is not None: + return _fix_model_name(family) + return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else listed_name def _is_listed_non_chat_model(model: str) -> bool: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 9114cc5d0b7..adda4eccd23 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2173,8 +2173,11 @@ def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deploym @pytest.mark.parametrize("model", ["azure/gpt-chat-latest", "azure/codex-mini"]) -def test_azure_openai_models_only_azure_lists_get_openai_pdf_pricing(model: str): +def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) @pytest.mark.parametrize( From 3aa68bb5cc6ab663d51cdeebe4c15506cb998c29 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:49:11 -0700 Subject: [PATCH 08/11] fix(token_counter): count reservation and project ITPM with the serving deployment's model Budget reservation and project ITPM counted with the requested group name, so a GPT-named group served by Claude reserved a PDF at OpenAI's price. They now count with each deployment's model minus its provider prefix, the name count_tokens uses, and take the largest. A custom OpenAI tokenizer with no model counts with cl100k again instead of raising, and listed Azure gpt- chat entries price as OpenAI. --- litellm/litellm_core_utils/token_counter.py | 7 +- .../hooks/parallel_request_limiter_v3.py | 19 +++++- .../spend_tracking/budget_reservation.py | 25 ++++++- litellm/proxy/spend_tracking/input_tokens.py | 16 ++++- .../litellm_core_utils/test_token_counter.py | 12 +++- .../hooks/test_parallel_request_limiter_v3.py | 30 +++++++++ .../spend_tracking/test_budget_reservation.py | 19 ++++++ .../proxy/spend_tracking/test_input_tokens.py | 66 +++++++++++++++++++ 8 files changed, 185 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7546565b2f3..97ef0fc84bf 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -483,11 +483,12 @@ def _cost_map_entry(model: str) -> Mapping[str, object]: def _is_azure_openai_chat_model(model: str) -> bool: - entry: Final = _cost_map_entry(f"azure/{_openai_deployment_name(model)}") + name: Final = _openai_deployment_name(model) + entry: Final = _cost_map_entry(f"azure/{name}") return ( entry.get("litellm_provider") == "azure" and entry.get("mode") in OPENAI_CHAT_MODES - and entry.get("supports_pdf_input") is True + and (entry.get("supports_pdf_input") is True or name.startswith("gpt-")) ) @@ -809,7 +810,7 @@ def _get_exact_count_function( return count_tokens if tokenizer_json["type"] == "openai_tokenizer": - encoding: Final = openai_tokenizer_encoding(model) + encoding: Final = openai_tokenizer_encoding(model or "") def encode_length(text: str) -> int: return _encoding_count(encoding, text) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index d858eff3348..c95ec14a943 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -71,6 +71,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.proxy.spend_tracking.input_tokens import models_counted_for from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, @@ -139,6 +140,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) +def _resolve_counted_models_via_proxy_router(model: str) -> Sequence[str]: + from litellm.proxy.proxy_server import llm_router + + return models_counted_for(model, llm_router) + + def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: from litellm.proxy.proxy_server import llm_router @@ -843,6 +850,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, + counted_models_resolver: Callable[[str], Sequence[str]] = _resolve_counted_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None ] = _resolve_ptu_team_ceiling_via_proxy_router, @@ -853,6 +861,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._counted_models_resolver = counted_models_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver @@ -3962,8 +3971,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): min_configured_tpm_limit=min_configured_otpm_limit, call_type=call_type, ) - raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + counted_models: Final = self._counted_models_resolver(requested_model) if requested_model else (None,) + raw_estimated_input_tokens: Final = max( + [ + await offload_token_count(self._estimate_precise_input_tokens)( + data=data, model=model, call_type=call_type + ) + for model in counted_models + ] ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 803df9f83ca..a621a09f1cb 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -7,6 +7,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain from types import MappingProxyType from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast @@ -39,7 +40,11 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model +from litellm.proxy.spend_tracking.input_tokens import ( + count_input_tokens, + count_input_tokens_for_model, + models_counted_for, +) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router @@ -1597,7 +1602,23 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) + counted_models: Final = MappingProxyType({model: models_counted_for(model, llm_router) for model in models}) + counts: Final = await count_input_tokens( + request_body=request_body, + raw_body=raw_body, + models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))), + ) + return MappingProxyType( + { + model: largest + for model, names in counted_models.items() + if (largest := _largest_count(counts, names)) is not None + } + ) + + +def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None: + return max((counts[name] for name in names if name in counts), default=None) def _estimate_input_tokens( diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 6c7083fb6db..67f3330d685 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -15,7 +15,7 @@ import asyncio import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger @@ -29,6 +29,9 @@ from litellm.rust_bridge.token_counter import ( rust_tokenizer, ) +if TYPE_CHECKING: + from litellm.router import Router + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -125,6 +128,17 @@ def _count_input_tokens_for_models( ) +def models_counted_for(model: str, llm_router: Router | None) -> tuple[str, ...]: + """The names a request for `model` is counted with: each deployment's model minus its provider prefix, the + name `/v1/messages/count_tokens` counts with, or `model` itself when no deployment serves it.""" + deployments: Final = (llm_router.get_model_list(model_name=model) or ()) if llm_router is not None else () + deployment_models: Final = (deployment["litellm_params"].get("model") for deployment in deployments) + served: Final = tuple( + dict.fromkeys(name.partition("/")[2] or name for name in deployment_models if isinstance(name, str) and name) + ) + return served or (model,) + + def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index adda4eccd23..e08910aae9f 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2172,7 +2172,9 @@ def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deploym ) -@pytest.mark.parametrize("model", ["azure/gpt-chat-latest", "azure/codex-mini"]) +@pytest.mark.parametrize( + "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"] +) def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( @@ -2397,6 +2399,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t } +def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None: + sample: Final = "A custom tokenizer passed with no model still counts." + + assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == ( + len(tiktoken.get_encoding("cl100k_base").encode(sample)) + ) + + def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: text_value: Final = "A tokenizer fallback should preserve the model encoding." custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value) diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 7af747fc26d..b2ed5e9cffb 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -8686,3 +8686,33 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit( await _rpm_request(limiter, cache, auth, "a2a/cheap") assert denied.value.status_code == 429 assert "expensive" in str(denied.value.detail) + + +@pytest.mark.asyncio +async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None: + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), + counted_models_resolver=lambda model: ("claude-haiku-5-5", "gpt-5.6-sol"), + ) + messages: Final = _one_page_pdf_message() + caller: Final = UserAPIKeyAuth( + api_key=hash_token("sk-pdf-itpm"), + project_id="pdf-itpm", + project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=cache, + data={"model": "gpt-4o", "messages": messages, "max_tokens": 10}, + call_type="acompletion", + ) + + stash: Final = get_request_stash() + assert stash is not None + claude_count: Final = handler._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5") + assert claude_count > handler._estimate_precise_input_tokens({"messages": messages}, "gpt-4o") + assert stash.itpm_reserved_tokens == claude_count diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..08b1cf318c2 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -426,3 +426,22 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe ) def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None: assert get_budget_window_start(window) == expected + + +@pytest.mark.asyncio +async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None: + from litellm.proxy.spend_tracking.budget_reservation import count_request_input_tokens + from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()} + router: Final = Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}] + ) + + counts: Final = await count_request_input_tokens( + request_body=request_body, route="/chat/completions", llm_router=router + ) + + assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} + assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o") diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index a1da6c79cd7..e003ddf7853 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 from types import MappingProxyType from typing import Final @@ -11,7 +12,9 @@ from litellm.proxy.spend_tracking.input_tokens import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, count_input_tokens, count_input_tokens_for_model, + models_counted_for, ) +from litellm.router import Router CL100K_MODEL: Final = "gpt-4" @@ -27,3 +30,66 @@ async def test_large_input_is_still_counted() -> None: assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) assert isinstance(counts, MappingProxyType) + + +def _one_page_pdf_message() -> list[dict[str, object]]: + stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET" + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> " + b"/Contents 5 0 R >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + ) + objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1)) + header: Final = b"%PDF-1.4\n" + offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n" + ).encode() + pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + return [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}}, + {"type": "text", "text": "Summarize the report."}, + ], + } + ] + + +def _router(*deployments: tuple[str, str]) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}} + for name, model in deployments + ] + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model"), + [("gpt-4o", "anthropic/claude-haiku-5-5"), ("pdf-reports", "azure/gpt-5.6-sol")], + ids=["gpt-named-group-served-by-claude", "custom-group-served-by-azure-gpt"], +) +def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, deployment_model: str) -> None: + request_body: Final = {"model": group, "messages": _one_page_pdf_message()} + counted_with: Final = deployment_model.partition("/")[2] + + assert models_counted_for(group, _router((group, deployment_model))) == (counted_with,) + assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( + request_body=request_body, model=group + ) + + +def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): + router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) + + assert models_counted_for("mixed", router) == ("claude-haiku-5-5", "gpt-5.6-sol") + assert models_counted_for("not-deployed", router) == ("not-deployed",) + assert models_counted_for("gpt-4o", None) == ("gpt-4o",) From 6ac4e23fbd8c018d312d23b610df01f8fb05b5da Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:28:32 -0700 Subject: [PATCH 09/11] perf(token_counter): find drawn PDF images with a byte scan instead of parsing every operator pypdf parsed every operator of a page's content stream to find the images it draws, so a compressed page of a million operators took about 30 seconds to price for an OpenAI model. A linear scan of the decoded stream for name Do and inline images finds the same draws. --- litellm/litellm_core_utils/token_counter.py | 46 +++++++++++++------ .../litellm_core_utils/test_token_counter.py | 26 +++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 314176767dc..9037e139d11 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,11 +3,12 @@ import base64 import io import math +import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import cached_property -from itertools import accumulate, chain +from itertools import accumulate, chain, islice from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast @@ -406,8 +407,7 @@ class _PdfBox(Protocol): class _PdfContent(Protocol): - @property - def operations(self) -> Sequence[tuple[object, bytes]]: ... + def get_data(self) -> bytes: ... class _PdfPage(Protocol): @@ -1043,26 +1043,44 @@ def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _ return _pdf_dictionary(_pdf_value(resources, "/XObject")) +_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " +_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" +_PDF_XOBJECT_DRAW: Final = re.compile( + rb"/([^" + _PDF_TOKEN_END + rb"]*)[" + _PDF_WHITESPACE + rb"]+Do(?![^" + _PDF_TOKEN_END + rb"])" +) +_PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: + try: + return raw.decode(charset) + except UnicodeDecodeError: + return None + + +def _pdf_name(escaped: bytes) -> str: + raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped) + decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS) + return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1")) + + def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary: - operations: Final = content.operations - operand_lists: Final = (operands for operands, operator in operations if operator == b"Do") - names: Final[Iterable[object]] = ( - cast(Sequence[object], operands)[0] # cast-ok: pypdf parses an operator's operands into a list - for operands in operand_lists - if isinstance(operands, list) and operands - ) + data: Final = content.get_data() + draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1) return _PdfContentSummary( - drawn_names=tuple(name for name in names if isinstance(name, str)), - draws_inline_image=any(operator == b"INLINE IMAGE" for _, operator in operations), + drawn_names=tuple(_pdf_name(draw[1]) for draw in draws), + draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None, ) def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None: - from pypdf.generic import ContentStream, StreamObject + from pypdf.generic import StreamObject if not isinstance(form, StreamObject): return None - return _pdf_content_summary(cast(_PdfContent, ContentStream(form, None))) # cast-ok: pypdf is untyped + return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 9166f48d466..77e67bbd23b 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2282,6 +2282,32 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_ ) == _rendered_page_cost() +@pytest.mark.timeout(10) +def test_openai_model_prices_a_page_of_a_million_drawing_operators_promptly(): + """A page of a million drawing operators and no image took about 30 seconds to price, since every operator was + parsed to find the images it draws, while the stream compresses into a request a few kilobytes long.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64(("",), drawing=b" q Q" * 1_000_000, page_resources="<< >>")) == 0 + + +@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) +def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str): + """A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf + decodes it, so an image drawn under such a name still prices the page as rendered.""" + model: Final = "gpt-4o" + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=(_ONE_PIXEL_IMAGE,), + drawing=f" {escaped_name} Do".encode(), + page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>", + ), + ) == _rendered_page_cost() + + @pytest.mark.timeout(10) def test_openai_model_reads_resources_shared_by_many_forms_once(): """3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once.""" From 317c4caffca39110e8721fdcdec1761d3eecb47f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:21:48 -0700 Subject: [PATCH 10/11] fix(token-counter): count each team deployment by its PDF pricing and honor base_model Budget reservation and the TPM limiter now look up the deployments that serve the caller's team, by base_model when set. A deployment that reads PDF pages and prices them unlike the group name counts with its own model, any other with the group name. A comment between an XObject name and Do now reads as whitespace, and the timing tests moved to CodSpeed benchmarks. --- litellm/litellm_core_utils/token_counter.py | 11 +- .../hooks/parallel_request_limiter_v3.py | 22 +-- .../spend_tracking/budget_reservation.py | 6 +- litellm/proxy/spend_tracking/input_tokens.py | 22 +-- .../test_token_counter_pdf_benchmarks.py | 126 ++++++++++++++++++ .../litellm_core_utils/test_token_counter.py | 64 ++++----- .../hooks/test_parallel_request_limiter_v3.py | 2 +- .../spend_tracking/test_budget_reservation.py | 2 +- .../proxy/spend_tracking/test_input_tokens.py | 56 +++++++- 9 files changed, 244 insertions(+), 67 deletions(-) create mode 100644 tests/benchmarks/test_token_counter_pdf_benchmarks.py diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 3472e9d2b56..69505309ef0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -709,7 +709,7 @@ class _MessageCountParams: @cached_property def pdf_pricing(self) -> PdfPagePricing: - return _pdf_page_pricing(self._model) + return pdf_page_pricing(self._model) def _is_openai_chat_model(model: str) -> bool: @@ -753,10 +753,15 @@ def _is_openai_or_azure_model(model: str) -> bool: return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) -def _pdf_page_pricing(model: str | None) -> PdfPagePricing: +def pdf_page_pricing(model: str | None) -> PdfPagePricing: return "openai" if _is_openai_or_azure_model(model or "") else "claude" +def reads_pdf_pages(model: str) -> bool: + entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2]) + return entry.get("supports_pdf_input") is True + + def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, @@ -1262,7 +1267,7 @@ def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _ _PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " _PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" _PDF_XOBJECT_DRAW: Final = re.compile( - rb"/([^" + _PDF_TOKEN_END + rb"]*)[" + _PDF_WHITESPACE + rb"]+Do(?![^" + _PDF_TOKEN_END + rb"])" + rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])" ) _PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) -def _resolve_counted_models_via_proxy_router(model: str) -> Sequence[str]: +def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]: from litellm.proxy.proxy_server import llm_router - return models_counted_for(model, llm_router) + return models_counted_for(model, llm_router, team_id) def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: @@ -885,7 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, - counted_models_resolver: Callable[[str], Sequence[str]] = _resolve_counted_models_via_proxy_router, + counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router, media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None @@ -4082,14 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type=call_type, include_media=False, ) - counted_models: Final = self._counted_models_resolver(requested_model) if requested_model else (None,) + counted_models: Final = ( + self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,) + ) raw_estimated_input_tokens: Final = max( - [ - await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=model, call_type=call_type + await asyncio.gather( + *( + offload_token_count(self._estimate_precise_input_tokens)( + data=data, model=model, call_type=call_type + ) + for model in counted_models ) - for model in counted_models - ] + ) ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 957700015dc..3f3256a059b 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -275,6 +275,7 @@ async def reserve_budget_for_request( request_body=request_body, route=route, llm_router=llm_router, + team_id=valid_token.team_id, raw_body=raw_body, ) @@ -1596,6 +1597,7 @@ async def count_request_input_tokens( request_body: dict, route: str, llm_router: Router | None, + team_id: str | None, raw_body: bytes | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. @@ -1604,7 +1606,9 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - counted_models: Final = MappingProxyType({model: models_counted_for(model, llm_router) for model in models}) + counted_models: Final = MappingProxyType( + {model: models_counted_for(model, llm_router, team_id) for model in models} + ) counts: Final = await count_input_tokens( request_body=request_body, raw_body=raw_body, diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 4c157d64640..35b0854d693 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, @@ -133,15 +134,20 @@ def _count_input_tokens_for_models( ) -def models_counted_for(model: str, llm_router: Router | None) -> tuple[str, ...]: - """The names a request for `model` is counted with: each deployment's model minus its provider prefix, the - name `/v1/messages/count_tokens` counts with, or `model` itself when no deployment serves it.""" - deployments: Final = (llm_router.get_model_list(model_name=model) or ()) if llm_router is not None else () - deployment_models: Final = (deployment["litellm_params"].get("model") for deployment in deployments) - served: Final = tuple( - dict.fromkeys(name.partition("/")[2] or name for name in deployment_models if isinstance(name, str) and name) +def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + """The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that + reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix, + the name `/v1/messages/count_tokens` counts with.""" + served: Final = ( + llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else () ) - return served or (model,) + return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,) + + +def _counted_name(model: str, served_model: str) -> str: + if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model): + return model + return served_model.partition("/")[2] or served_model def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..b2a86459438 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,126 @@ +"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed. + +Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size.""" + +import base64 +from itertools import accumulate, chain +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import token_counter + +TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." + + +def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str: + escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(), + *xobjects, + ) + header: Final = b"%PDF-1.4\n" + objects: Final = tuple( + f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1) + ) + offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1]))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n" + ).encode() + return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + + +def _form(resources_number: int, drawing: bytes) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _count_pdf(pdf_base64: str) -> int: + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}} + return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}]) + + +def _page_of_drawing_operators() -> str: + return _pdf_base64("", "<< >>", b" q Q" * 50_000) + + +def _page_of_forms_sharing_resources() -> str: + form_count: Final = 300 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count)) + return _pdf_base64( + TEXT_LINE, + "<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>", + b" q 10 0 0 10 0 0 cm /X0 Do Q", + (forms[0], shared_resources, *forms), + ) + + +def _page_of_parents_sharing_a_child_form() -> str: + parent_count: Final = 100 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child)) + + +PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators() +PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources() +PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form() +BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b"")) +TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b"")) +LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000 +NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8 +UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE) + + +@pytest.mark.benchmark +def test_pdf_page_of_drawing_operators_and_no_image() -> None: + assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None: + assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None: + assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_token_counter_long_unknown_model_name() -> None: + assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d260d56d765..607f8240905 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -29,7 +29,11 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -2283,13 +2287,19 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_ ) == _rendered_page_cost() -@pytest.mark.timeout(10) -def test_openai_model_prices_a_page_of_a_million_drawing_operators_promptly(): - """A page of a million drawing operators and no image took about 30 seconds to price, since every operator was - parsed to find the images it draws, while the stream compresses into a request a few kilobytes long.""" +@pytest.mark.parametrize( + ("drawing", "draws_image"), + [(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)], + ids=["comment-before-do", "do-inside-a-comment"], +) +def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool): + """A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after + it nor draws anything itself.""" model: Final = "gpt-4o" - assert _pdf_cost(model, _pdf_base64(("",), drawing=b" q Q" * 1_000_000, page_resources="<< >>")) == 0 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == ( + _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13) + ) @pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) @@ -2309,32 +2319,16 @@ def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str) ) == _rendered_page_cost() -@pytest.mark.timeout(10) -def test_openai_model_reads_resources_shared_by_many_forms_once(): - """3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once.""" +@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)]) +def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered( + parent_count: int, draws_too_many: bool +): + """A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects + between them, so a page of forms nested to blow up the walk cannot stall the request.""" model: Final = "gpt-4o" - form_count: Final = 3000 - shared_resources_number: Final = 7 - shared_resources: Final = ( - "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" - ).encode() - forms: Final = tuple(_form_xobject(shared_resources_number, b"/F0 Do") for _ in range(form_count)) - - assert _pdf_cost( - model, _pdf_base64((_REVENUE_LINE,), xobjects=(forms[0], shared_resources, *forms), drawing=_DRAW_X0) - ) == (_text_cost(model, _REVENUE_LINE) + 13) - - -@pytest.mark.timeout(10) -def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly(): - """1,000 forms that each list their own /Resources and draw one shared 4,000-operator child took about 45 - seconds to price, since the child was parsed and walked again under every parent. A page now prices as if it - drew an image once its forms draw more than a fixed number of XObjects, and each form is parsed once.""" - model: Final = "gpt-4o" - parent_count: Final = 1000 first: Final = 6 child_number: Final = first + 2 * parent_count - child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2)) child: Final = ( f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + child_drawing @@ -2358,7 +2352,7 @@ def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly(): _pdf_base64( (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources ), - ) == _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13) _NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 @@ -2461,16 +2455,6 @@ def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_mode ) -@pytest.mark.timeout(10) -def test_a_very_long_unknown_model_name_counts_promptly(): - """A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried.""" - long_name: Final = "unknown-" + "a-" * 200_000 - - assert token_counter_new(model=long_name, text=_NON_LATIN_SAMPLE) == token_counter_new( - model="my-private-model", text=_NON_LATIN_SAMPLE - ) - - @pytest.mark.parametrize( "model", [ diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 856e7aa7ed5..c82ecb72b09 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -9106,7 +9106,7 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep cache: Final = DualCache() handler: Final = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(cache), - counted_models_resolver=lambda model: ("claude-haiku-5-5", "gpt-5.6-sol"), + counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"), ) messages: Final = _one_page_pdf_message() caller: Final = UserAPIKeyAuth( diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 08b1cf318c2..65a1969ece6 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -440,7 +440,7 @@ async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_g ) counts: Final = await count_request_input_tokens( - request_body=request_body, route="/chat/completions", llm_router=router + request_body=request_body, route="/chat/completions", llm_router=router, team_id=None ) assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index 1236fb516a5..148001be32b 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -82,18 +82,66 @@ def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, dep request_body: Final = {"model": group, "messages": _one_page_pdf_message()} counted_with: Final = deployment_model.partition("/")[2] - assert models_counted_for(group, _router((group, deployment_model))) == (counted_with,) + assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,) assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( request_body=request_body, model=group ) +@pytest.mark.parametrize( + ("group", "deployment_model"), + [ + ("my-claude", "anthropic/claude-haiku-5-5"), + ("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), + ("my-cohere", "cohere/command-r-plus"), + ("gpt-4o", "cohere/command-r-plus"), + ], + ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"], +) +def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group( + group: str, deployment_model: str +) -> None: + assert models_counted_for(group, _router((group, deployment_model)), None) == (group,) + + +def test_a_custom_named_deployment_counts_with_its_base_model(): + router: Final = Router( + model_list=[ + { + "model_name": "reports", + "litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"}, + "model_info": {"base_model": "azure/gpt-5.6-sol"}, + }, + {"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}}, + ] + ) + + assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",) + assert models_counted_for("drafts", router, None) == ("drafts",) + + def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) - assert models_counted_for("mixed", router) == ("claude-haiku-5-5", "gpt-5.6-sol") - assert models_counted_for("not-deployed", router) == ("not-deployed",) - assert models_counted_for("gpt-4o", None) == ("gpt-4o",) + assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol") + assert models_counted_for("not-deployed", router, None) == ("not-deployed",) + assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",) + + +def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): + router: Final = Router( + model_list=[ + {"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}, + { + "model_name": "reports-team-a", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "reports"}, + }, + ] + ) + + assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",) + assert models_counted_for("reports", router, "team-b") == ("reports",) def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: From 7729925ad73aabc25848fabaaa32c782460932af Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:02:35 -0700 Subject: [PATCH 11/11] fix(benchmarks): install pypdf in the benchmark environment The PDF token-count benchmarks parse pages with pypdf, which only the proxy-runtime extra installed, so CodSpeed took the no-parser path. --- pyproject.toml | 1 + uv.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 00cdb1587aa..6acd863701a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -312,6 +312,7 @@ benchmarks = [ "pytest-codspeed==4.3.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", + "pypdf>=6.21.2,<7.0", ] [build-system] diff --git a/uv.lock b/uv.lock index b40490112cf..87a86b6812f 100644 --- a/uv.lock +++ b/uv.lock @@ -5118,6 +5118,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -5347,6 +5348,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.21.2,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ]