From 3feb5f27dfb5e082e52d4ed7b60ea8e0079592d7 Mon Sep 17 00:00:00 2001 From: RachelHuangZW Date: Fri, 9 Oct 2026 17:23:14 -0400 Subject: [PATCH 01/21] fix(token_counter): bound the cost of pricing a base64 PDF Pricing an inline PDF parsed and text-extracted every page on the token count path, so its cost grew with page count and, superlinearly, with a single page's content stream. A small compressed upload could hold one of the four token-count slots for seconds to minutes, stalling every other request that needs a count on that worker Text is now extracted from at most EXTRAPOLATION_SAMPLES evenly spaced pages and scaled to the page count, and a page whose content stream exceeds PDF_PAGE_MAX_EXACT_CONTENT_BYTES is priced at one token per content byte instead of being extracted. Every page's rendering tokens are still counted --- litellm/litellm_core_utils/token_counter.py | 55 +++++++++++++++++-- .../litellm_core_utils/test_token_counter.py | 40 +++++++++++++- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7ac3e9c80c5..cd737e8e326 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -6,7 +6,7 @@ import math import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast import anyio import anyio.lowlevel @@ -344,6 +344,7 @@ Type for a function that counts tokens in a string. """ EXTRAPOLATION_SAMPLES: Final = 16 +PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") @@ -818,6 +819,43 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) +class _PdfContentStream(Protocol): + def get_data(self) -> bytes: ... + + +class _PdfMediaBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfPage(Protocol): + @property + def mediabox(self) -> _PdfMediaBox: ... + + def get_contents(self) -> _PdfContentStream | None: ... + + def extract_text(self) -> str: ... + + +def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: + if page_count <= EXTRAPOLATION_SAMPLES: + return tuple(range(page_count)) + return tuple((page_count - 1) * index // (EXTRAPOLATION_SAMPLES - 1) for index in range(EXTRAPOLATION_SAMPLES)) + + +def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + """A page whose content stream is too large to extract cheaply is priced at one token per content byte, + which no byte-level tokenizer undercuts.""" + contents: Final = page.get_contents() + content_bytes: Final = len(contents.get_data()) if contents is not None else 0 + if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return content_bytes + return count_function(page.extract_text() or "") + + def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None @@ -827,12 +865,17 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + pages: Final = cast( # cast-ok: pypdf is an optional extra the type-check env does not install + Sequence[_PdfPage], PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))).pages ) + rendering_tokens: Final = sum( + _anthropic_rendered_page_image_tokens(page.mediabox.width, page.mediabox.height) for page in pages + ) + sampled_pages: Final = tuple(pages[index] for index in _evenly_spaced_page_indices(len(pages))) + if not sampled_pages: + return rendering_tokens + sampled_text_tokens: Final = sum(_pdf_page_text_tokens(page, count_function) for page in sampled_pages) + return rendering_tokens + round(sampled_text_tokens * len(pages) / len(sampled_pages)) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index a646135efc3..e18acc00db4 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -27,9 +27,12 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import PDF_DATA_URL_PREFIX, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( + EXTRAPOLATION_SAMPLES, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES, + _count_inline_pdf_tokens, _encoding_count, _get_exact_count_function, _get_extrapolating_count_function, @@ -1608,6 +1611,41 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 +def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): + count_function: Final = MagicMock(side_effect=len) + forty_pages: Final = _pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",) * 40) + + _count_inline_pdf_tokens(PDF_DATA_URL_PREFIX + forty_pages, count_function) + + assert count_function.call_count <= EXTRAPOLATION_SAMPLES + + +def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): + count_function: Final = MagicMock(side_effect=len) + sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " + page_text: Final = sentence * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(sentence)) + + priced: Final = _count_inline_pdf_tokens(PDF_DATA_URL_PREFIX + _pdf_base64((page_text,)), count_function) + + assert priced is not None and priced >= len(page_text) + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + +def test_long_pdf_of_identical_pages_prices_each_page_like_a_one_page_pdf(): + prompt: Final = {"type": "text", "text": "Summarize this file."} + page_text: Final = "Revenue grew eleven percent while churn fell to two percent." + page_count: Final = 3 * EXTRAPOLATION_SAMPLES + 1 + base: Final = _count_user_content([prompt]) + + def pdf_cost(pages: int) -> int: + document: Final = {"type": "document", "source": _pdf_source(_pdf_base64((page_text,) * pages))} + return _count_user_content([prompt, document]) - base + + assert pdf_cost(page_count) == page_count * pdf_cost(1) + + def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest.MonkeyPatch): prompt: Final = {"type": "text", "text": "Summarize this file."} source: Final = _pdf_source(_pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",))) From c1c3d5e05207665543d4c3806df7986998d656fb Mon Sep 17 00:00:00 2001 From: RachelHuangZW Date: Fri, 9 Oct 2026 18:15:48 -0400 Subject: [PATCH 02/21] test(token_counter): count pdf text extraction directly with injected pages Pricing a page list now lives in _price_pdf_pages, which takes any object shaped like a pypdf page. The bounded-work tests pass fake pages and count extract_text calls, so extracting every page and sampling afterwards fails them, as does extracting a page before checking its content size --- litellm/litellm_core_utils/token_counter.py | 20 ++++++++------ .../litellm_core_utils/test_token_counter.py | 26 ++++++++++++++++--- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index cd737e8e326..da21e45d67c 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -856,6 +856,17 @@ def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) return count_function(page.extract_text() or "") +def _price_pdf_pages(pages: Sequence[_PdfPage], count_function: TokenCounterFunction) -> int: + rendering_tokens: Final = sum( + _anthropic_rendered_page_image_tokens(page.mediabox.width, page.mediabox.height) for page in pages + ) + sampled_pages: Final = tuple(pages[index] for index in _evenly_spaced_page_indices(len(pages))) + if not sampled_pages: + return rendering_tokens + sampled_text_tokens: Final = sum(_pdf_page_text_tokens(page, count_function) for page in sampled_pages) + return rendering_tokens + round(sampled_text_tokens * len(pages) / len(sampled_pages)) + + def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None @@ -868,14 +879,7 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction pages: Final = cast( # cast-ok: pypdf is an optional extra the type-check env does not install Sequence[_PdfPage], PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))).pages ) - rendering_tokens: Final = sum( - _anthropic_rendered_page_image_tokens(page.mediabox.width, page.mediabox.height) for page in pages - ) - sampled_pages: Final = tuple(pages[index] for index in _evenly_spaced_page_indices(len(pages))) - if not sampled_pages: - return rendering_tokens - sampled_text_tokens: Final = sum(_pdf_page_text_tokens(page, count_function) for page in sampled_pages) - return rendering_tokens + round(sampled_text_tokens * len(pages) / len(sampled_pages)) + return _price_pdf_pages(pages, count_function) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index e18acc00db4..a5f19698cca 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from concurrent.futures import Future, wait from itertools import accumulate, chain from pathlib import Path +from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -34,6 +35,7 @@ from litellm.litellm_core_utils.token_counter import ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES, _count_inline_pdf_tokens, _encoding_count, + _price_pdf_pages, _get_exact_count_function, _get_extrapolating_count_function, _get_tiktoken_count_function, @@ -1611,13 +1613,29 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 +def _fake_pdf_page(content_bytes: int) -> SimpleNamespace: + return SimpleNamespace( + mediabox=SimpleNamespace(width=612.0, height=792.0), + get_contents=lambda: SimpleNamespace(get_data=lambda: b"x" * content_bytes), + extract_text=MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), + ) + + def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): - count_function: Final = MagicMock(side_effect=len) - forty_pages: Final = _pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",) * 40) + pages: Final = tuple(_fake_pdf_page(content_bytes=1_000) for _ in range(40)) - _count_inline_pdf_tokens(PDF_DATA_URL_PREFIX + forty_pages, count_function) + _price_pdf_pages(pages, len) - assert count_function.call_count <= EXTRAPOLATION_SAMPLES + assert sum(page.extract_text.call_count for page in pages) <= EXTRAPOLATION_SAMPLES + + +def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): + page: Final = _fake_pdf_page(content_bytes=2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + + priced: Final = _price_pdf_pages((page,), len) + + page.extract_text.assert_not_called() + assert priced >= 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): From 89793b5adfd658356dd66da96d3b61314dcee035 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 18:31:33 +0000 Subject: [PATCH 03/21] fix(token_counter): price oversized PDF pages by their string operands, not every content byte Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 16 +++++++--- .../litellm_core_utils/test_token_counter.py | 32 +++++++++++++++++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index da21e45d67c..bfea2023e3e 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,6 +3,7 @@ import base64 import io import math +import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from itertools import accumulate @@ -345,6 +346,7 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 +PDF_STRING_OPERAND: Final = re.compile(rb"\((?:\\.|[^\\()])*\)|<[0-9A-Fa-f\s]+>") T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") @@ -846,13 +848,17 @@ def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: return tuple((page_count - 1) * index // (EXTRAPOLATION_SAMPLES - 1) for index in range(EXTRAPOLATION_SAMPLES)) +def _pdf_string_operand_bytes(content: bytes) -> int: + return sum(match.end() - match.start() for match in PDF_STRING_OPERAND.finditer(content)) + + def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - """A page whose content stream is too large to extract cheaply is priced at one token per content byte, - which no byte-level tokenizer undercuts.""" + """A page whose content stream is too large to extract cheaply is priced at one token per byte of its string + operands, which no byte-level tokenizer undercuts.""" contents: Final = page.get_contents() - content_bytes: Final = len(contents.get_data()) if contents is not None else 0 - if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - return content_bytes + content: Final = contents.get_data() if contents is not None else b"" + if len(content) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return _pdf_string_operand_bytes(content) return count_function(page.extract_text() or "") diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index a5f19698cca..91ec84417a6 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1613,10 +1613,11 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 -def _fake_pdf_page(content_bytes: int) -> SimpleNamespace: +def _fake_pdf_page(content_bytes: int | bytes) -> SimpleNamespace: + content: Final = b"x" * content_bytes if isinstance(content_bytes, int) else content_bytes return SimpleNamespace( mediabox=SimpleNamespace(width=612.0, height=792.0), - get_contents=lambda: SimpleNamespace(get_data=lambda: b"x" * content_bytes), + get_contents=lambda: SimpleNamespace(get_data=lambda: content), extract_text=MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), ) @@ -1630,7 +1631,10 @@ def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): - page: Final = _fake_pdf_page(content_bytes=2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + string_operand: Final = b"(Revenue grew eleven percent)" + text_operator: Final = string_operand + b" Tj\n" + content: Final = text_operator * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(string_operand) + 1) + page: Final = _fake_pdf_page(content_bytes=content) priced: Final = _price_pdf_pages((page,), len) @@ -1638,6 +1642,28 @@ def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): assert priced >= 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES +def test_oversized_pdf_page_is_priced_from_its_string_operands_without_extracting_text(): + content: Final = b"BT /F1 9 Tf (Figure 3) Tj ET\n" + b"10 10 m 200 300 l S\n" * 10_000 + page: Final = _fake_pdf_page(content_bytes=content) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + blank_page.extract_text.return_value = "" + + priced: Final = _price_pdf_pages((page,), len) - _price_pdf_pages((blank_page,), len) + + page.extract_text.assert_not_called() + assert priced <= len(b"(Figure 3)") + + +def test_unterminated_pdf_literal_string_is_priced_in_linear_time(): + page: Final = _fake_pdf_page(content_bytes=b"(" * (4 * 1024 * 1024)) + + started: Final = time.perf_counter() + _price_pdf_pages((page,), len) + elapsed: Final = time.perf_counter() - started + + assert elapsed < 1.0 + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From 0aeb504c8d42eb0c7cc3deb5e7f561e659982f6c Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 19:15:32 +0000 Subject: [PATCH 04/21] fix(token_counter): count Form XObject content toward the PDF page extraction limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 96 +++++++++++- .../litellm_core_utils/test_token_counter.py | 143 ++++++++++++++++++ 2 files changed, 232 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bfea2023e3e..c880208fb62 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -6,8 +6,9 @@ import math import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from functools import reduce from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable import anyio import anyio.lowlevel @@ -825,6 +826,26 @@ class _PdfContentStream(Protocol): def get_data(self) -> bytes: ... +@runtime_checkable +class _PdfResourceDictionary(Protocol): + def get(self, key: str) -> object: ... + + +@runtime_checkable +class _PdfXObjectDictionary(_PdfResourceDictionary, Protocol): + def values(self) -> Iterable[object]: ... + + +@runtime_checkable +class _PdfFormXObject(_PdfResourceDictionary, Protocol): + def get_data(self) -> bytes: ... + + +@runtime_checkable +class _PdfIndirectReference(Protocol): + def get_object(self) -> object: ... + + class _PdfMediaBox(Protocol): @property def width(self) -> float: ... @@ -833,7 +854,7 @@ class _PdfMediaBox(Protocol): def height(self) -> float: ... -class _PdfPage(Protocol): +class _PdfPage(_PdfResourceDictionary, Protocol): @property def mediabox(self) -> _PdfMediaBox: ... @@ -842,6 +863,67 @@ class _PdfPage(Protocol): def extract_text(self) -> str: ... +def _pdf_resolve_object(value: object) -> object: + return value.get_object() if isinstance(value, _PdfIndirectReference) else value + + +def _pdf_collect_form_xobject_streams( + xobjects: Iterable[object], + visited: frozenset[int], +) -> tuple[tuple[bytes, ...], frozenset[int]]: + return reduce(_pdf_collect_form_xobject, xobjects, ((), visited)) + + +def _pdf_collect_form_xobject( + collected: tuple[tuple[bytes, ...], frozenset[int]], + xobject: object, +) -> tuple[tuple[bytes, ...], frozenset[int]]: + collected_streams, visited = collected + xobject_streams, updated_visited = _pdf_form_xobject_content_streams(xobject, visited) + return (collected_streams + xobject_streams, updated_visited) + + +def _pdf_form_xobject_content_streams( + xobject: object, + visited: frozenset[int], +) -> tuple[tuple[bytes, ...], frozenset[int]]: + resolved: Final = _pdf_resolve_object(xobject) + identity: Final = id(resolved) + if identity in visited: + return (), visited + visited_with_xobject: Final = visited | {identity} + if not isinstance(resolved, _PdfFormXObject): + return (), visited_with_xobject + if resolved.get("/Subtype") != "/Form": + return (), visited_with_xobject + content: Final = resolved.get_data() + resources: Final = _pdf_resolve_object(resolved.get("/Resources")) + if not isinstance(resources, _PdfResourceDictionary): + return (content,), visited_with_xobject + xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) + if not isinstance(xobjects, _PdfXObjectDictionary): + return (content,), visited_with_xobject + nested_streams, final_visited = _pdf_collect_form_xobject_streams(xobjects.values(), visited_with_xobject) + return ((content, *nested_streams), final_visited) + + +def _pdf_page_form_xobject_content_streams(page: _PdfPage) -> tuple[bytes, ...]: + resources: Final = _pdf_resolve_object(page.get("/Resources")) + if not isinstance(resources, _PdfResourceDictionary): + return () + xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) + if not isinstance(xobjects, _PdfXObjectDictionary): + return () + streams, _ = _pdf_collect_form_xobject_streams(xobjects.values(), frozenset()) + return streams + + +def _pdf_page_content_streams(page: _PdfPage) -> tuple[bytes, ...]: + contents: Final = page.get_contents() + content: Final = contents.get_data() if contents is not None else b"" + return (content, *_pdf_page_form_xobject_content_streams(page)) + + def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: if page_count <= EXTRAPOLATION_SAMPLES: return tuple(range(page_count)) @@ -853,12 +935,12 @@ def _pdf_string_operand_bytes(content: bytes) -> int: def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - """A page whose content stream is too large to extract cheaply is priced at one token per byte of its string + """A page whose content streams are too large to extract cheaply is priced at one token per byte of their string operands, which no byte-level tokenizer undercuts.""" - contents: Final = page.get_contents() - content: Final = contents.get_data() if contents is not None else b"" - if len(content) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - return _pdf_string_operand_bytes(content) + content_streams: Final = _pdf_page_content_streams(page) + content_bytes: Final = sum(len(content) for content in content_streams) + if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return sum(_pdf_string_operand_bytes(content) for content in content_streams) return count_function(page.extract_text() or "") diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 91ec84417a6..511d419dc90 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1563,6 +1563,50 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() +def _pdf_stream(content: bytes, dictionary: bytes = b"") -> bytes: + dictionary_entries: Final = b" " + dictionary if dictionary else b"" + return ( + b"<< /Length " + + str(len(content)).encode() + + dictionary_entries + + b" >>\nstream\n" + + content + + b"\nendstream" + ) + + +def _pdf_data_url_with_xobjects( + page_content: bytes, + page_xobjects: bytes, + xobject_objects: tuple[bytes, ...], +) -> str: + object_bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 3 0 R >> /XObject " + + page_xobjects + + b" >> /Contents 5 0 R >>", + _pdf_stream(page_content), + *xobject_objects, + ) + object_chunks: Final = tuple( + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) + ) + header: Final = b"%PDF-1.4\n" + offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) + xref: Final = f"xref\n0 {len(object_chunks) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(object_chunks) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(chunk) for chunk in object_chunks)}\n%%EOF\n" + ).encode() + return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() + + def _pdf_source(pdf_base64: str) -> dict[str, str]: return {"type": "base64", "media_type": "application/pdf", "data": pdf_base64} @@ -1615,10 +1659,15 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st def _fake_pdf_page(content_bytes: int | bytes) -> SimpleNamespace: content: Final = b"x" * content_bytes if isinstance(content_bytes, int) else content_bytes + + def get_page_resource(_: str) -> None: + return None + return SimpleNamespace( mediabox=SimpleNamespace(width=612.0, height=792.0), get_contents=lambda: SimpleNamespace(get_data=lambda: content), extract_text=MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), + get=get_page_resource, ) @@ -1664,6 +1713,100 @@ def test_unterminated_pdf_literal_string_is_priced_in_linear_time(): assert elapsed < 1.0 +def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): + string_operand: Final = b"(form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + form_content: Final = text_operator * repetitions + form_object: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= len(string_operand) * repetitions + + +def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text(): + string_operand: Final = b"(nested form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + nested_content: Final = text_operator * repetitions + nested_form: Final = _pdf_stream( + nested_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + form: Final = _pdf_stream( + b"/Fm2 Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + b"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm2 7 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form, nested_form), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= len(string_operand) * repetitions + + +def test_cyclic_pdf_form_xobject_is_priced(): + path_operator: Final = b"10 10 m 200 300 l S\n" + form_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + b"/Fm1 Do\n" + form: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /XObject << /Fm1 6 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form,), + ) + + priced: Final = _count_inline_pdf_tokens(data_url, len) + + assert priced is not None + + +def test_pdf_image_xobject_does_not_trigger_the_content_limit(): + page_text: Final = b"BT /F1 12 Tf 20 760 Td (small page text) Tj ET\n" + image_data: Final = b"\x00" * (1024 * 1024) + image: Final = _pdf_stream( + image_data, + b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 " + b"/ColorSpace /DeviceRGB /BitsPerComponent 8", + ) + data_url: Final = _pdf_data_url_with_xobjects( + page_text, + b"<< /Im1 6 0 R >>", + (image,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert any("small page text" in call.args[0] for call in count_function.call_args_list) + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From add181ed0d8b93c8585619c45221e92ef21a7efc Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 19:44:03 +0000 Subject: [PATCH 05/21] fix(token_counter): follow only drawn Form XObjects and scan PDF strings in linear time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 68 ++++++---- .../litellm_core_utils/test_token_counter.py | 128 ++++++++++++++++++ 2 files changed, 169 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index c880208fb62..ec782a12191 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -347,7 +347,8 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 -PDF_STRING_OPERAND: Final = re.compile(rb"\((?:\\.|[^\\()])*\)|<[0-9A-Fa-f\s]+>") +PDF_STRING_OPERAND: Final = re.compile(rb"\(+(?:\\[\s\S]|[^\\()])*\)?|<[0-9A-Fa-f\s]+>") +PDF_XOBJECT_INVOCATION: Final = re.compile(rb"/([^\s/\[\]()<>{}%]+)\s+Do\b") T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") @@ -867,61 +868,74 @@ def _pdf_resolve_object(value: object) -> object: return value.get_object() if isinstance(value, _PdfIndirectReference) else value -def _pdf_collect_form_xobject_streams( - xobjects: Iterable[object], - visited: frozenset[int], -) -> tuple[tuple[bytes, ...], frozenset[int]]: - return reduce(_pdf_collect_form_xobject, xobjects, ((), visited)) - - -def _pdf_collect_form_xobject( +def _pdf_collect_invoked_form_xobject( collected: tuple[tuple[bytes, ...], frozenset[int]], - xobject: object, + invocation: re.Match[bytes], + xobjects: _PdfXObjectDictionary, + inherited_resources: _PdfResourceDictionary, ) -> tuple[tuple[bytes, ...], frozenset[int]]: collected_streams, visited = collected - xobject_streams, updated_visited = _pdf_form_xobject_content_streams(xobject, visited) + xobject_name: Final = "/" + invocation.group(1).decode("latin-1") + xobject: Final = _pdf_resolve_object(xobjects.get(xobject_name)) + xobject_streams, updated_visited = _pdf_form_xobject_content_streams( + xobject, + inherited_resources, + visited, + ) return (collected_streams + xobject_streams, updated_visited) +def _pdf_collect_invoked_form_xobjects( + content: bytes, + resources: _PdfResourceDictionary, + visited: frozenset[int], +) -> tuple[tuple[bytes, ...], frozenset[int]]: + xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) + if not isinstance(xobjects, _PdfXObjectDictionary): + return (), visited + + def collect( + collected: tuple[tuple[bytes, ...], frozenset[int]], + invocation: re.Match[bytes], + ) -> tuple[tuple[bytes, ...], frozenset[int]]: + return _pdf_collect_invoked_form_xobject(collected, invocation, xobjects, resources) + + return reduce(collect, PDF_XOBJECT_INVOCATION.finditer(content), ((), visited)) + + def _pdf_form_xobject_content_streams( xobject: object, + inherited_resources: _PdfResourceDictionary, visited: frozenset[int], ) -> tuple[tuple[bytes, ...], frozenset[int]]: resolved: Final = _pdf_resolve_object(xobject) identity: Final = id(resolved) if identity in visited: return (), visited - visited_with_xobject: Final = visited | {identity} if not isinstance(resolved, _PdfFormXObject): - return (), visited_with_xobject + return (), visited if resolved.get("/Subtype") != "/Form": - return (), visited_with_xobject + return (), visited + visited_with_xobject: Final = visited | {identity} content: Final = resolved.get_data() - resources: Final = _pdf_resolve_object(resolved.get("/Resources")) - if not isinstance(resources, _PdfResourceDictionary): - return (content,), visited_with_xobject - xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) - if not isinstance(xobjects, _PdfXObjectDictionary): - return (content,), visited_with_xobject - nested_streams, final_visited = _pdf_collect_form_xobject_streams(xobjects.values(), visited_with_xobject) + form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) + resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources + nested_streams, final_visited = _pdf_collect_invoked_form_xobjects(content, resources, visited_with_xobject) return ((content, *nested_streams), final_visited) -def _pdf_page_form_xobject_content_streams(page: _PdfPage) -> tuple[bytes, ...]: +def _pdf_page_form_xobject_content_streams(page: _PdfPage, content: bytes) -> tuple[bytes, ...]: resources: Final = _pdf_resolve_object(page.get("/Resources")) if not isinstance(resources, _PdfResourceDictionary): return () - xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) - if not isinstance(xobjects, _PdfXObjectDictionary): - return () - streams, _ = _pdf_collect_form_xobject_streams(xobjects.values(), frozenset()) + streams, _ = _pdf_collect_invoked_form_xobjects(content, resources, frozenset()) return streams def _pdf_page_content_streams(page: _PdfPage) -> tuple[bytes, ...]: contents: Final = page.get_contents() content: Final = contents.get_data() if contents is not None else b"" - return (content, *_pdf_page_form_xobject_content_streams(page)) + return (content, *_pdf_page_form_xobject_content_streams(page, content)) def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 511d419dc90..e2e507e3858 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1607,6 +1607,52 @@ def _pdf_data_url_with_xobjects( return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() +def _pdf_data_url_with_shared_xobjects( + page_contents: tuple[bytes, ...], + xobject_objects: tuple[bytes, ...], +) -> str: + page_count: Final = len(page_contents) + page_object_numbers: Final = tuple(5 + index for index in range(page_count)) + content_object_numbers: Final = tuple(5 + page_count + index for index in range(page_count)) + first_xobject_object_number: Final = 5 + 2 * page_count + page_references: Final = b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) + xobject_references: Final = b" ".join( + f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() + for index in range(len(xobject_objects)) + ) + page_objects: Final = tuple( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources 4 0 R /Contents " + + f"{content_object} 0 R >>".encode() + for content_object in content_object_numbers + ) + content_objects: Final = tuple(_pdf_stream(content) for content in page_contents) + object_bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [" + + page_references + + f"] /Count {page_count} >>".encode(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Font << /F1 3 0 R >> /XObject << " + xobject_references + b" >> >>", + *page_objects, + *content_objects, + *xobject_objects, + ) + object_chunks: Final = tuple( + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) + ) + header: Final = b"%PDF-1.4\n" + offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) + xref: Final = f"xref\n0 {len(object_chunks) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(object_chunks) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(chunk) for chunk in object_chunks)}\n%%EOF\n" + ).encode() + return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() + + def _pdf_source(pdf_base64: str) -> dict[str, str]: return {"type": "base64", "media_type": "application/pdf", "data": pdf_base64} @@ -1807,6 +1853,88 @@ def test_pdf_image_xobject_does_not_trigger_the_content_limit(): assert any("small page text" in call.args[0] for call in count_function.call_args_list) +def test_pdf_pages_with_shared_resources_price_only_the_form_they_draw(): + page_count: Final = 20 + string_operand: Final = b"(text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2_800 // len(string_operand) + page_contents: Final = tuple(f"/Fm{index} Do\n".encode() for index in range(page_count)) + form_objects: Final = tuple( + _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + for _ in range(page_count) + ) + data_url: Final = _pdf_data_url_with_shared_xobjects(page_contents, form_objects) + blank_data_url: Final = _pdf_data_url_with_shared_xobjects((b"",) * page_count, ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) + string_operand_bytes: Final = page_count * len(string_operand) * repetitions + + assert priced is not None + assert rendering_tokens is not None + assert priced <= string_operand_bytes * 3 // 2 + rendering_tokens + + +def test_pdf_form_without_resources_inherits_page_xobjects(): + string_operand: Final = b"(inherited form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + nested_form_content: Final = text_operator * repetitions + inherited_form: Final = _pdf_stream( + nested_form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + unused_form: Final = _pdf_stream( + nested_form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + form_without_resources: Final = _pdf_stream( + b"/Fm2 Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792]", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R /Fm2 7 0 R /Fm3 8 0 R >>", + (form_without_resources, inherited_form, unused_form), + ) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) + string_operand_bytes: Final = len(string_operand) * repetitions + + assert priced is not None + assert rendering_tokens is not None + assert priced >= string_operand_bytes + assert priced <= string_operand_bytes * 3 // 2 + rendering_tokens + + +def test_oversized_pdf_string_operand_scan_is_linear_after_escaped_parentheses(): + path_operator: Final = b"10 10 m 20 20 l S\n" + padding_repetitions: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + padding_bytes: Final = padding_repetitions * len(path_operator) + padding: Final = path_operator * padding_repetitions + b" " * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES - padding_bytes) + content: Final = padding + b"%" + b"\\(" * 8_192 + path_operator * 1_000 + page: Final = _fake_pdf_page(content_bytes=content) + + started: Final = time.perf_counter() + _price_pdf_pages((page,), len) + elapsed: Final = time.perf_counter() - started + + assert elapsed < 1.0 + + +def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): + escaped_newlines: Final = b"a\\\n" * 10_000 + literal_content: Final = b"(" + escaped_newlines + b"a" * 50_000 + b") Tj\n" + page: Final = _fake_pdf_page(content_bytes=literal_content) + + priced: Final = _price_pdf_pages((page,), len) + + assert priced >= 60_000 + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From d7dff37a3d2271fff74a3f11af50d23ae4fb9dce Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 20:51:51 +0000 Subject: [PATCH 06/21] fix(token_counter): count repeated form draws and tokenize sampled PDF string text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 176 ++++++++++-------- .../test_token_counter_pdf_benchmarks.py | 40 ++++ .../litellm_core_utils/test_token_counter.py | 129 ++++++++++--- 3 files changed, 246 insertions(+), 99 deletions(-) create mode 100644 tests/benchmarks/test_token_counter_pdf_benchmarks.py diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ec782a12191..b2a3d39bef2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -6,9 +6,8 @@ import math import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from functools import reduce -from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable +from itertools import accumulate, chain, islice +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast, runtime_checkable import anyio import anyio.lowlevel @@ -347,8 +346,10 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 -PDF_STRING_OPERAND: Final = re.compile(rb"\(+(?:\\[\s\S]|[^\\()])*\)?|<[0-9A-Fa-f\s]+>") -PDF_XOBJECT_INVOCATION: Final = re.compile(rb"/([^\s/\[\]()<>{}%]+)\s+Do\b") +PDF_STRING_OPERAND: Final = re.compile(rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?|<[0-9A-Fa-f\s]+>") +PDF_XOBJECT_INVOCATION: Final = re.compile( + rb"%[^\r\n]*(?:\r\n?|\n|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:\r\n?|\n|\Z))+Do\b" +) T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") @@ -832,11 +833,6 @@ class _PdfResourceDictionary(Protocol): def get(self, key: str) -> object: ... -@runtime_checkable -class _PdfXObjectDictionary(_PdfResourceDictionary, Protocol): - def values(self) -> Iterable[object]: ... - - @runtime_checkable class _PdfFormXObject(_PdfResourceDictionary, Protocol): def get_data(self) -> bytes: ... @@ -868,74 +864,88 @@ def _pdf_resolve_object(value: object) -> object: return value.get_object() if isinstance(value, _PdfIndirectReference) else value -def _pdf_collect_invoked_form_xobject( - collected: tuple[tuple[bytes, ...], frozenset[int]], - invocation: re.Match[bytes], - xobjects: _PdfXObjectDictionary, - inherited_resources: _PdfResourceDictionary, -) -> tuple[tuple[bytes, ...], frozenset[int]]: - collected_streams, visited = collected - xobject_name: Final = "/" + invocation.group(1).decode("latin-1") - xobject: Final = _pdf_resolve_object(xobjects.get(xobject_name)) - xobject_streams, updated_visited = _pdf_form_xobject_content_streams( - xobject, - inherited_resources, - visited, - ) - return (collected_streams + xobject_streams, updated_visited) +_PdfStreamMetrics: TypeAlias = tuple[int, int, tuple[tuple[int, bytes], ...], bool] -def _pdf_collect_invoked_form_xobjects( - content: bytes, - resources: _PdfResourceDictionary, - visited: frozenset[int], -) -> tuple[tuple[bytes, ...], frozenset[int]]: - xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) - if not isinstance(xobjects, _PdfXObjectDictionary): - return (), visited +def _pdf_decode_name(name: bytes) -> str: + def decode_escape(match: re.Match[bytes]) -> bytes: + return bytes((int(match.group(1), 16),)) - def collect( - collected: tuple[tuple[bytes, ...], frozenset[int]], - invocation: re.Match[bytes], - ) -> tuple[tuple[bytes, ...], frozenset[int]]: - return _pdf_collect_invoked_form_xobject(collected, invocation, xobjects, resources) - - return reduce(collect, PDF_XOBJECT_INVOCATION.finditer(content), ((), visited)) + return "/" + re.sub(rb"#([0-9A-Fa-f]{2})", decode_escape, name).decode("latin-1") -def _pdf_form_xobject_content_streams( - xobject: object, - inherited_resources: _PdfResourceDictionary, - visited: frozenset[int], -) -> tuple[tuple[bytes, ...], frozenset[int]]: - resolved: Final = _pdf_resolve_object(xobject) - identity: Final = id(resolved) - if identity in visited: - return (), visited - if not isinstance(resolved, _PdfFormXObject): - return (), visited - if resolved.get("/Subtype") != "/Form": - return (), visited - visited_with_xobject: Final = visited | {identity} - content: Final = resolved.get_data() - form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) - resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources - nested_streams, final_visited = _pdf_collect_invoked_form_xobjects(content, resources, visited_with_xobject) - return ((content, *nested_streams), final_visited) +def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None: + name: Final = invocation.group(1) + return _pdf_decode_name(name) if name is not None else None -def _pdf_page_form_xobject_content_streams(page: _PdfPage, content: bytes) -> tuple[bytes, ...]: +def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: + form_data_memo: Final[dict[int, bytes]] = {} # mutable-ok: page-local cache avoids repeated stream decoding + form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks + + def stream_metrics( + content: bytes, + resources: _PdfResourceDictionary, + ancestors: frozenset[int], + ) -> _PdfStreamMetrics: + xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) + if not isinstance(xobjects, _PdfResourceDictionary): + return len(content), _pdf_string_operand_bytes(content), (), False + invoked_names: Final = tuple( + name + for invocation in PDF_XOBJECT_INVOCATION.finditer(content) + if (name := _pdf_invocation_name(invocation)) is not None + ) + form_metrics: Final = tuple( + form_metrics_for_xobject( + _pdf_resolve_object(xobjects.get(name)), + resources, + ancestors, + ) + for name in invoked_names + ) + return ( + len(content) + sum(metric[0] for metric in form_metrics), + _pdf_string_operand_bytes(content) + sum(metric[1] for metric in form_metrics), + tuple(dict.fromkeys(chain.from_iterable(metric[2] for metric in form_metrics))), + any(metric[3] for metric in form_metrics), + ) + + def form_metrics_for_xobject( + xobject: object, + inherited_resources: _PdfResourceDictionary, + ancestors: frozenset[int], + ) -> _PdfStreamMetrics: + resolved: Final = _pdf_resolve_object(xobject) + identity: Final = id(resolved) + if identity in ancestors: + return 0, 0, (), True + if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form": + return 0, 0, (), False + cached: Final = form_metrics_memo.get(identity) + if cached is not None: + return cached + cached_content: Final = form_data_memo.get(identity) + content: Final = cached_content if cached_content is not None else resolved.get_data() + if cached_content is None: + form_data_memo[identity] = content + form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) + resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources + metrics: Final = stream_metrics(content, resources, ancestors | {identity}) + result: Final = ( + metrics[0], + metrics[1], + tuple(dict.fromkeys(((identity, _pdf_string_operand_sample(content)), *metrics[2]))), + metrics[3], + ) + if not result[3]: + form_metrics_memo[identity] = result + return result + resources: Final = _pdf_resolve_object(page.get("/Resources")) if not isinstance(resources, _PdfResourceDictionary): - return () - streams, _ = _pdf_collect_invoked_form_xobjects(content, resources, frozenset()) - return streams - - -def _pdf_page_content_streams(page: _PdfPage) -> tuple[bytes, ...]: - contents: Final = page.get_contents() - content: Final = contents.get_data() if contents is not None else b"" - return (content, *_pdf_page_form_xobject_content_streams(page, content)) + return len(page_content), _pdf_string_operand_bytes(page_content), (), False + return stream_metrics(page_content, resources, frozenset()) def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: @@ -948,13 +958,31 @@ def _pdf_string_operand_bytes(content: bytes) -> int: return sum(match.end() - match.start() for match in PDF_STRING_OPERAND.finditer(content)) +def _pdf_string_operand_sample(content: bytes) -> bytes: + return bytes( + islice( + chain.from_iterable(match.group(0) for match in PDF_STRING_OPERAND.finditer(content)), + PDF_PAGE_MAX_EXACT_CONTENT_BYTES, + ) + ) + + +def _pdf_bounded_operand_sample(samples: Iterable[bytes]) -> bytes: + return bytes(islice(chain.from_iterable(samples), PDF_PAGE_MAX_EXACT_CONTENT_BYTES)) + + def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - """A page whose content streams are too large to extract cheaply is priced at one token per byte of their string - operands, which no byte-level tokenizer undercuts.""" - content_streams: Final = _pdf_page_content_streams(page) - content_bytes: Final = sum(len(content) for content in content_streams) + """Oversized pages use bounded string-operand sampling and scale its token estimate to the full page.""" + contents: Final = page.get_contents() + content: Final = contents.get_data() if contents is not None else b"" + content_bytes, operand_bytes, form_samples, _ = _pdf_page_content_metrics(page, content) if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - return sum(_pdf_string_operand_bytes(content) for content in content_streams) + sample: Final = _pdf_bounded_operand_sample( + (_pdf_string_operand_sample(content), *(form_sample for _, form_sample in form_samples)) + ) + if not sample: + return 0 + return round(count_function(sample.decode("latin-1")) * operand_bytes / len(sample)) return count_function(page.extract_text() or "") diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..e1f55bf6282 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,40 @@ +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import _price_pdf_pages + + +def _pdf_page(content: bytes) -> SimpleNamespace: + return SimpleNamespace( + mediabox=SimpleNamespace(width=612.0, height=792.0), + get_contents=lambda: SimpleNamespace(get_data=lambda: content), + extract_text=lambda: "", + get=lambda key: {"/XObject": {}} if key == "/Resources" else None, + ) + + +@pytest.mark.benchmark +def test_unterminated_pdf_literal_string_scan(): + page: Final = _pdf_page(b"(" * (4 * 1024 * 1024)) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +def test_pdf_string_operand_scan_after_escaped_parentheses(): + path_operator: Final = b"10 10 m 20 20 l S\n" + padding_repetitions: Final = 64 * 1024 // len(path_operator) + padding_bytes: Final = padding_repetitions * len(path_operator) + padding: Final = path_operator * padding_repetitions + b" " * (64 * 1024 - padding_bytes) + page: Final = _pdf_page(padding + b"%" + b"\\(" * 8_192 + path_operator * 1_000) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +def test_pdf_invocation_scan_ignores_unterminated_comments(): + page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4)) + + _price_pdf_pages((page,), len) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index e2e507e3858..447bf0f2d33 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1749,16 +1749,6 @@ def test_oversized_pdf_page_is_priced_from_its_string_operands_without_extractin assert priced <= len(b"(Figure 3)") -def test_unterminated_pdf_literal_string_is_priced_in_linear_time(): - page: Final = _fake_pdf_page(content_bytes=b"(" * (4 * 1024 * 1024)) - - started: Final = time.perf_counter() - _price_pdf_pages((page,), len) - elapsed: Final = time.perf_counter() - started - - assert elapsed < 1.0 - - def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): string_operand: Final = b"(form text)" text_operator: Final = string_operand + b" Tj\n" @@ -1784,6 +1774,30 @@ def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): assert priced >= len(string_operand) * repetitions +def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): + string_operand: Final = b"(repeated form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 1_024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n" * 1_000, + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= len(string_operand) * repetitions * 1_000 + + def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text(): string_operand: Final = b"(nested form text)" text_operator: Final = string_operand + b" Tj\n" @@ -1814,6 +1828,32 @@ def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text assert priced >= len(string_operand) * repetitions +def test_exponential_pdf_form_draws_avoid_full_text_extraction(): + depth: Final = 20 + form_objects: Final = tuple( + _pdf_stream( + (f"/Fm{index + 1} Do\n" * 2).encode() if index < depth - 1 else b"(leaf) Tj\n", + ( + f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + f"/Resources << /XObject << /Fm{index + 1} {8 + index} 0 R >> >>" + ).encode() + if index < depth - 1 + else b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + ) + for index in range(depth) + ) + data_url: Final = _pdf_data_url_with_shared_xobjects((b"/Fm0 Do\n",), form_objects) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= 2 ** (depth - 1) + + def test_cyclic_pdf_form_xobject_is_priced(): path_operator: Final = b"10 10 m 200 300 l S\n" form_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + b"/Fm1 Do\n" @@ -1910,21 +1950,6 @@ def test_pdf_form_without_resources_inherits_page_xobjects(): assert priced <= string_operand_bytes * 3 // 2 + rendering_tokens -def test_oversized_pdf_string_operand_scan_is_linear_after_escaped_parentheses(): - path_operator: Final = b"10 10 m 20 20 l S\n" - padding_repetitions: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) - padding_bytes: Final = padding_repetitions * len(path_operator) - padding: Final = path_operator * padding_repetitions + b" " * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES - padding_bytes) - content: Final = padding + b"%" + b"\\(" * 8_192 + path_operator * 1_000 - page: Final = _fake_pdf_page(content_bytes=content) - - started: Final = time.perf_counter() - _price_pdf_pages((page,), len) - elapsed: Final = time.perf_counter() - started - - assert elapsed < 1.0 - - def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): escaped_newlines: Final = b"a\\\n" * 10_000 literal_content: Final = b"(" + escaped_newlines + b"a" * 50_000 + b") Tj\n" @@ -1935,6 +1960,60 @@ def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): assert priced >= 60_000 +def test_nested_pdf_literal_string_keeps_its_full_operand(): + content: Final = b"(a(b)" + b"x" * 100_000 + b") Tj\n" + page: Final = _fake_pdf_page(content_bytes=content) + + priced: Final = _price_pdf_pages((page,), len) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced >= 100_000 + + +def test_pdf_form_invocations_allow_comments_and_escaped_names(): + string_operand: Final = b"(named form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 1_024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + ) + path_operator: Final = b"10 10 m 20 20 l S\n" + page_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + ( + b"/Fm1 %comment\n Do\n/F#6d1 Do\n" + ) + data_url: Final = _pdf_data_url_with_xobjects( + page_content, + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced >= 2 * len(string_operand) * repetitions + + +def test_oversized_pdf_text_uses_a_sampled_token_estimate(): + string_operand: Final = b"(Revenue grew eleven percent)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=text_operator * repetitions) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + def four_chars_per_token(value: str) -> int: + return len(value) // 4 + + page_priced: Final = _price_pdf_pages((page,), four_chars_per_token) # pyright: ignore[reportPrivateUsage] # PDF regression test + blank_page_priced: Final = _price_pdf_pages( + (blank_page,), four_chars_per_token + ) # pyright: ignore[reportPrivateUsage] # PDF regression test + priced: Final = page_priced - blank_page_priced + expected: Final = len(string_operand) * repetitions // 4 + + assert expected * 4 // 5 <= priced <= expected * 6 // 5 + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From 628754bd7a0e2ddc077a0784e2238a5bf64b620a Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 20:58:58 +0000 Subject: [PATCH 07/21] fix(token_counter): memoize cyclic PDF form walks and skip strings when finding form draws Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 28 ++++------ .../litellm_core_utils/test_token_counter.py | 51 +++++++++++++++++++ 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index b2a3d39bef2..020acc3b9e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -346,9 +346,10 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 -PDF_STRING_OPERAND: Final = re.compile(rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?|<[0-9A-Fa-f\s]+>") +PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?" +PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]+>") PDF_XOBJECT_INVOCATION: Final = re.compile( - rb"%[^\r\n]*(?:\r\n?|\n|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:\r\n?|\n|\Z))+Do\b" + PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:\r\n?|\n|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:\r\n?|\n|\Z))+Do\b" ) T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -864,7 +865,7 @@ def _pdf_resolve_object(value: object) -> object: return value.get_object() if isinstance(value, _PdfIndirectReference) else value -_PdfStreamMetrics: TypeAlias = tuple[int, int, tuple[tuple[int, bytes], ...], bool] +_PdfStreamMetrics: TypeAlias = tuple[int, int, tuple[tuple[int, bytes], ...]] def _pdf_decode_name(name: bytes) -> str: @@ -880,7 +881,6 @@ def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None: def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: - form_data_memo: Final[dict[int, bytes]] = {} # mutable-ok: page-local cache avoids repeated stream decoding form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks def stream_metrics( @@ -890,7 +890,7 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream ) -> _PdfStreamMetrics: xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) if not isinstance(xobjects, _PdfResourceDictionary): - return len(content), _pdf_string_operand_bytes(content), (), False + return len(content), _pdf_string_operand_bytes(content), () invoked_names: Final = tuple( name for invocation in PDF_XOBJECT_INVOCATION.finditer(content) @@ -908,7 +908,6 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream len(content) + sum(metric[0] for metric in form_metrics), _pdf_string_operand_bytes(content) + sum(metric[1] for metric in form_metrics), tuple(dict.fromkeys(chain.from_iterable(metric[2] for metric in form_metrics))), - any(metric[3] for metric in form_metrics), ) def form_metrics_for_xobject( @@ -919,16 +918,13 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream resolved: Final = _pdf_resolve_object(xobject) identity: Final = id(resolved) if identity in ancestors: - return 0, 0, (), True + return 0, 0, () if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form": - return 0, 0, (), False + return 0, 0, () cached: Final = form_metrics_memo.get(identity) if cached is not None: return cached - cached_content: Final = form_data_memo.get(identity) - content: Final = cached_content if cached_content is not None else resolved.get_data() - if cached_content is None: - form_data_memo[identity] = content + content: Final = resolved.get_data() form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) @@ -936,15 +932,13 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream metrics[0], metrics[1], tuple(dict.fromkeys(((identity, _pdf_string_operand_sample(content)), *metrics[2]))), - metrics[3], ) - if not result[3]: - form_metrics_memo[identity] = result + form_metrics_memo[identity] = result return result resources: Final = _pdf_resolve_object(page.get("/Resources")) if not isinstance(resources, _PdfResourceDictionary): - return len(page_content), _pdf_string_operand_bytes(page_content), (), False + return len(page_content), _pdf_string_operand_bytes(page_content), () return stream_metrics(page_content, resources, frozenset()) @@ -975,7 +969,7 @@ def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) """Oversized pages use bounded string-operand sampling and scale its token estimate to the full page.""" contents: Final = page.get_contents() content: Final = contents.get_data() if contents is not None else b"" - content_bytes, operand_bytes, form_samples, _ = _pdf_page_content_metrics(page, content) + content_bytes, operand_bytes, form_samples = _pdf_page_content_metrics(page, content) if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: sample: Final = _pdf_bounded_operand_sample( (_pdf_string_operand_sample(content), *(form_sample for _, form_sample in form_samples)) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 447bf0f2d33..28eedcb7b52 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1854,6 +1854,33 @@ def test_exponential_pdf_form_draws_avoid_full_text_extraction(): assert priced >= 2 ** (depth - 1) +@pytest.mark.timeout(10) +def test_cyclic_exponential_pdf_form_walk_is_memoized(): + depth: Final = 25 + + def cyclic_form(index: int) -> bytes: + next_index: Final = (index + 1) % depth + next_form_content: Final = f"/Fm{next_index} Do\n/Fm{next_index} Do\n".encode() + form_content: Final = b"(x) Tj\n" + next_form_content + form_dictionary: Final = ( + f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + f"/Resources << /XObject << /Fm{next_index} {7 + next_index} 0 R >> >>" + ).encode() + return _pdf_stream(form_content, form_dictionary) + + form_objects: Final = tuple(cyclic_form(index) for index in range(depth)) + data_url: Final = _pdf_data_url_with_shared_xobjects((b"/Fm0 Do\n",), form_objects) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced > 0 + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + def test_cyclic_pdf_form_xobject_is_priced(): path_operator: Final = b"10 10 m 200 300 l S\n" form_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + b"/Fm1 Do\n" @@ -1994,6 +2021,30 @@ def test_pdf_form_invocations_allow_comments_and_escaped_names(): assert priced >= 2 * len(string_operand) * repetitions +def test_pdf_percent_inside_literal_does_not_hide_form_invocation(): + string_operand: Final = b"(form text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 100 * 1024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"(50%) Tj /Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced >= len(string_operand) * repetitions + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + def test_oversized_pdf_text_uses_a_sampled_token_estimate(): string_operand: Final = b"(Revenue grew eleven percent)" text_operator: Final = string_operand + b" Tj\n" From 48cc412a4795617420e83e37a3847d541dab4310 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 21:25:36 +0000 Subject: [PATCH 08/21] fix(token_counter): skip PDF comments and inline images and decode sampled strings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 59 ++++++++-- .../test_token_counter_pdf_benchmarks.py | 15 +++ .../litellm_core_utils/test_token_counter.py | 110 ++++++++++++++++-- 3 files changed, 162 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 88bfea8f04a..abdf5fd392d 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -351,9 +351,11 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?" -PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]+>") +PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*") +PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s\S])") +PDF_INLINE_IMAGE: Final = re.compile(rb"\bBI\b[\s\S]*?(?:\bID\s[\s\S]*?(?:\sEI\b|\Z)|\Z)") PDF_XOBJECT_INVOCATION: Final = re.compile( - PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:\r\n?|\n|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:\r\n?|\n|\Z))+Do\b" + PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" ) T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -966,7 +968,7 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream cached: Final = form_metrics_memo.get(identity) if cached is not None: return cached - content: Final = resolved.get_data() + content: Final = PDF_INLINE_IMAGE.sub(b" ", resolved.get_data()) form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) @@ -991,15 +993,50 @@ def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: def _pdf_string_operand_bytes(content: bytes) -> int: - return sum(match.end() - match.start() for match in PDF_STRING_OPERAND.finditer(content)) + return sum(len(operand) for operand in _pdf_decoded_string_operands(content)) def _pdf_string_operand_sample(content: bytes) -> bytes: - return bytes( - islice( - chain.from_iterable(match.group(0) for match in PDF_STRING_OPERAND.finditer(content)), - PDF_PAGE_MAX_EXACT_CONTENT_BYTES, - ) + return bytes(islice(chain.from_iterable(_pdf_decoded_string_operands(content)), PDF_PAGE_MAX_EXACT_CONTENT_BYTES)) + + +def _pdf_decode_literal_escape(match: re.Match[bytes]) -> bytes: + escape: Final = match.group(1) + if escape is None: + return b"" + if escape == b"n": + return b"\n" + if escape == b"r": + return b"\r" + if escape == b"t": + return b"\t" + if escape == b"b": + return b"\b" + if escape == b"f": + return b"\f" + if escape in (b"(", b")", b"\\"): + return escape + if escape in (b"\r", b"\n", b"\r\n"): + return b"" + if escape[:1] in b"01234567": + return bytes((int(escape, 8) & 0xFF,)) + return escape + + +def _pdf_decode_string_operand(operand: bytes) -> bytes: + if operand.startswith(b"<"): + digits: Final = re.sub(rb"\s+", b"", operand[1:-1]) + padded_digits: Final = digits if len(digits) % 2 == 0 else digits + b"0" + return bytes.fromhex(padded_digits.decode("ascii")) + literal: Final = operand[1:-1] if operand.endswith(b")") else operand[1:] + return PDF_LITERAL_ESCAPE.sub(_pdf_decode_literal_escape, literal) + + +def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: + return ( + _pdf_decode_string_operand(match.group(0)) + for match in PDF_STRING_OPERAND.finditer(content) + if not match.group(0).startswith(b"%") ) @@ -1008,9 +1045,9 @@ def _pdf_bounded_operand_sample(samples: Iterable[bytes]) -> bytes: def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - """Oversized pages use bounded string-operand sampling and scale its token estimate to the full page.""" + """Oversized pages use bounded decoded string samples scaled to the full decoded operand text.""" contents: Final = page.get_contents() - content: Final = contents.get_data() if contents is not None else b"" + content: Final = PDF_INLINE_IMAGE.sub(b" ", contents.get_data() if contents is not None else b"") content_bytes, operand_bytes, form_samples = _pdf_page_content_metrics(page, content) if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: sample: Final = _pdf_bounded_operand_sample( diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index e1f55bf6282..41ea9c8d676 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -38,3 +38,18 @@ def test_pdf_invocation_scan_ignores_unterminated_comments(): page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4)) _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "comments", + [ + b"%comment\r\n" * 23, + b"% a \r\n" * 40, + ], + ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], +) +def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): + page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET") + + _price_pdf_pages((page,), len) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index e5d0e65b7ee..48d385c78eb 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -3,8 +3,10 @@ import asyncio import base64 import importlib +import io import json import os +import random import subprocess import sys import threading @@ -18,6 +20,7 @@ from typing import Final from unittest.mock import MagicMock import anyio.to_thread +from pypdf import PdfReader import pytest import tiktoken from tokenizers import Regex, Tokenizer, models, pre_tokenizers @@ -1962,11 +1965,11 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 -def _fake_pdf_page(content_bytes: int | bytes) -> SimpleNamespace: +def _fake_pdf_page(content_bytes: int | bytes, include_xobjects: bool = False) -> SimpleNamespace: content: Final = b"x" * content_bytes if isinstance(content_bytes, int) else content_bytes - def get_page_resource(_: str) -> None: - return None + def get_page_resource(key: str) -> object | None: + return {"/XObject": {}} if include_xobjects and key == "/Resources" else None return SimpleNamespace( mediabox=SimpleNamespace(width=612.0, height=792.0), @@ -1987,7 +1990,8 @@ def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): string_operand: Final = b"(Revenue grew eleven percent)" text_operator: Final = string_operand + b" Tj\n" - content: Final = text_operator * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(string_operand) + 1) + decoded_operand_bytes: Final = len(string_operand) - 2 + content: Final = text_operator * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // decoded_operand_bytes + 1) page: Final = _fake_pdf_page(content_bytes=content) priced: Final = _price_pdf_pages((page,), len) @@ -2030,7 +2034,7 @@ def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES ) - assert priced >= len(string_operand) * repetitions + assert priced >= (len(string_operand) - 2) * repetitions def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): @@ -2054,7 +2058,7 @@ def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES ) - assert priced >= len(string_operand) * repetitions * 1_000 + assert priced >= (len(string_operand) - 2) * repetitions * 1_000 def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text(): @@ -2084,7 +2088,7 @@ def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES ) - assert priced >= len(string_operand) * repetitions + assert priced >= (len(string_operand) - 2) * repetitions def test_exponential_pdf_form_draws_avoid_full_text_extraction(): @@ -2196,7 +2200,7 @@ def test_pdf_pages_with_shared_resources_price_only_the_form_they_draw(): blank_data_url: Final = _pdf_data_url_with_shared_xobjects((b"",) * page_count, ()) priced: Final = _count_inline_pdf_tokens(data_url, len) rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) - string_operand_bytes: Final = page_count * len(string_operand) * repetitions + string_operand_bytes: Final = page_count * (len(string_operand) - 2) * repetitions assert priced is not None assert rendering_tokens is not None @@ -2228,7 +2232,7 @@ def test_pdf_form_without_resources_inherits_page_xobjects(): blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) priced: Final = _count_inline_pdf_tokens(data_url, len) rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) - string_operand_bytes: Final = len(string_operand) * repetitions + string_operand_bytes: Final = (len(string_operand) - 2) * repetitions assert priced is not None assert rendering_tokens is not None @@ -2277,7 +2281,7 @@ def test_pdf_form_invocations_allow_comments_and_escaped_names(): priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test assert priced is not None - assert priced >= 2 * len(string_operand) * repetitions + assert priced >= 2 * (len(string_operand) - 2) * repetitions def test_pdf_percent_inside_literal_does_not_hide_form_invocation(): @@ -2298,12 +2302,96 @@ def test_pdf_percent_inside_literal_does_not_hide_form_invocation(): priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test assert priced is not None - assert priced >= len(string_operand) * repetitions + assert priced >= (len(string_operand) - 2) * repetitions assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES ) +@pytest.mark.parametrize( + "comments", + [ + b"%comment\r\n" * 23, + b"% a \r\n" * 40, + ], + ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], +) +def test_pdf_invocation_comment_separators_are_priceable(comments: bytes): + content: Final = b"BT /F1 " + comments + b"12 Tf (hello) Tj ET" + page: Final = _fake_pdf_page(content_bytes=content, include_xobjects=True) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + priced: Final = _price_pdf_pages((page,), len) + expected: Final = _price_pdf_pages((blank_page,), len) + + assert priced == expected + + +@pytest.mark.parametrize("inline_image_in_form", [False, True], ids=["page-stream", "form-stream"]) +def test_pdf_inline_image_bytes_do_not_push_content_over_the_limit(inline_image_in_form: bool): + image_data: Final = random.Random(42).randbytes(270_000) + inline_image: Final = ( + b"BI /W 270000 /H 1 /BPC 8 /CS /RGB ID " + image_data + b" EI\nBT /F1 9 Tf (Hello world) Tj ET\n" + ) + form: Final = _pdf_stream( + inline_image, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + page_content: Final = b"/Fm1 Do\n" if inline_image_in_form else inline_image + page_xobjects: Final = b"<< /Fm1 6 0 R >>" if inline_image_in_form else b"<< >>" + form_objects: Final = (form,) if inline_image_in_form else () + data_url: Final = _pdf_data_url_with_xobjects(page_content, page_xobjects, form_objects) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + page: Final = PdfReader(io.BytesIO(pdf_bytes)).pages[0] + page_text: Final = page.extract_text() or "" + + def count_function(value: str) -> int: + return len(value) + + expected: Final = count_function(page_text) + count_calls: Final = MagicMock(side_effect=count_function) + priced: Final = _count_inline_pdf_tokens(data_url, count_calls) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_function) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == expected + count_calls.assert_called_once_with(page_text) + + +def test_pdf_comments_are_not_counted_as_string_operands(): + comment: Final = b"% (lots of text)\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(comment) + 1 + content: Final = comment * repetitions + b"(x) Tj\n" + page: Final = _fake_pdf_page(content_bytes=content) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + blank_page.extract_text.return_value = "" + + priced: Final = _price_pdf_pages((page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + + assert priced - blank_price == len(b"x") + + +def test_oversized_hex_pdf_strings_price_like_literal_text(): + literal_operand: Final = b"(Hello world)" + hex_operand: Final = b"<48656c6c6f20776f726c64>" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(hex_operand + b" Tj\n") + 1 + literal_page: Final = _fake_pdf_page(content_bytes=(literal_operand + b" Tj\n") * repetitions) + hex_page: Final = _fake_pdf_page(content_bytes=(hex_operand + b" Tj\n") * repetitions) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + literal_price: Final = _price_pdf_pages((literal_page,), len) + hex_price: Final = _price_pdf_pages((hex_page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + literal_text_tokens: Final = literal_price - blank_price + hex_text_tokens: Final = hex_price - blank_price + + assert hex_text_tokens * 5 <= literal_text_tokens * 6 + assert literal_text_tokens * 5 <= hex_text_tokens * 6 + + def test_oversized_pdf_text_uses_a_sampled_token_estimate(): string_operand: Final = b"(Revenue grew eleven percent)" text_operator: Final = string_operand + b" Tj\n" From 09762cf984ddcdeacf4d3a070cbf551e67cba21a Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 21:33:06 +0000 Subject: [PATCH 09/21] test(token_counter): mark parser benchmarks as performance-only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/benchmarks/test_token_counter_pdf_benchmarks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index 41ea9c8d676..b6f61393327 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -16,14 +16,14 @@ def _pdf_page(content: bytes) -> SimpleNamespace: @pytest.mark.benchmark -def test_unterminated_pdf_literal_string_scan(): +def test_unterminated_pdf_literal_string_scan(): # test-quality-ok: CodSpeed performance benchmark page: Final = _pdf_page(b"(" * (4 * 1024 * 1024)) _price_pdf_pages((page,), len) @pytest.mark.benchmark -def test_pdf_string_operand_scan_after_escaped_parentheses(): +def test_pdf_string_operand_scan_after_escaped_parentheses(): # test-quality-ok: CodSpeed performance benchmark path_operator: Final = b"10 10 m 20 20 l S\n" padding_repetitions: Final = 64 * 1024 // len(path_operator) padding_bytes: Final = padding_repetitions * len(path_operator) @@ -34,7 +34,7 @@ def test_pdf_string_operand_scan_after_escaped_parentheses(): @pytest.mark.benchmark -def test_pdf_invocation_scan_ignores_unterminated_comments(): +def test_pdf_invocation_scan_ignores_unterminated_comments(): # test-quality-ok: CodSpeed performance benchmark page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4)) _price_pdf_pages((page,), len) @@ -49,7 +49,7 @@ def test_pdf_invocation_scan_ignores_unterminated_comments(): ], ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], ) -def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): +def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): # test-quality-ok: CodSpeed performance benchmark page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET") _price_pdf_pages((page,), len) From 8ad6b5a6e686946a818ee120e30bb6c56f53286d Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 21:42:48 +0000 Subject: [PATCH 10/21] fix(token_counter): skip strings and comments when stripping PDF inline images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 17 ++++++++++++--- .../test_token_counter_pdf_benchmarks.py | 7 +++++++ .../litellm_core_utils/test_token_counter.py | 21 +++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index abdf5fd392d..e1fde4b3ed4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -353,7 +353,11 @@ PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?" PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*") PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s\S])") -PDF_INLINE_IMAGE: Final = re.compile(rb"\bBI\b[\s\S]*?(?:\bID\s[\s\S]*?(?:\sEI\b|\Z)|\Z)") +PDF_INLINE_IMAGE: Final = re.compile( + PDF_LITERAL_STRING_OPERAND + + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=\s)" + + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:\sEI(?![^\s\[/<(%])|\Z)|\Z))" +) PDF_XOBJECT_INVOCATION: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" ) @@ -924,6 +928,10 @@ def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None: return _pdf_decode_name(name) if name is not None else None +def _pdf_inline_image_replacement(match: re.Match[bytes]) -> bytes: + return b" " if match.group("inline_image") is not None else match.group(0) + + def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks @@ -968,7 +976,7 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream cached: Final = form_metrics_memo.get(identity) if cached is not None: return cached - content: Final = PDF_INLINE_IMAGE.sub(b" ", resolved.get_data()) + content: Final = PDF_INLINE_IMAGE.sub(_pdf_inline_image_replacement, resolved.get_data()) form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) @@ -1047,7 +1055,10 @@ def _pdf_bounded_operand_sample(samples: Iterable[bytes]) -> bytes: def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: """Oversized pages use bounded decoded string samples scaled to the full decoded operand text.""" contents: Final = page.get_contents() - content: Final = PDF_INLINE_IMAGE.sub(b" ", contents.get_data() if contents is not None else b"") + content: Final = PDF_INLINE_IMAGE.sub( + _pdf_inline_image_replacement, + contents.get_data() if contents is not None else b"", + ) content_bytes, operand_bytes, form_samples = _pdf_page_content_metrics(page, content) if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: sample: Final = _pdf_bounded_operand_sample( diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index b6f61393327..ff9a7218d8e 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -40,6 +40,13 @@ def test_pdf_invocation_scan_ignores_unterminated_comments(): # test-quality-ok _price_pdf_pages((page,), len) +@pytest.mark.benchmark +def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"BI " * 1_000_000) + + _price_pdf_pages((page,), len) + + @pytest.mark.benchmark @pytest.mark.parametrize( "comments", diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 48d385c78eb..21913ca689c 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2360,6 +2360,27 @@ def test_pdf_inline_image_bytes_do_not_push_content_over_the_limit(inline_image_ count_calls.assert_called_once_with(page_text) +@pytest.mark.parametrize( + "prefix", + [b"(BI) Tj\n", b"/BI 1 Tf\n", b"/BIfoo 1 Tf\n"], + ids=["literal-string", "inline-image-name", "longer-name"], +) +def test_pdf_inline_image_operator_is_not_found_inside_strings_or_names(prefix: bytes): + string_operand: Final = b"(sample text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _price_pdf_pages((page,), count_function) + + page.extract_text.assert_not_called() + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + def test_pdf_comments_are_not_counted_as_string_operands(): comment: Final = b"% (lots of text)\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(comment) + 1 From 7f17f6154445c97808dfc473e15a92fb88eda854 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 21:47:56 +0000 Subject: [PATCH 11/21] fix(token_counter): only strip terminated PDF inline images Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 6 ++++-- .../litellm_core_utils/test_token_counter.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index e1fde4b3ed4..8b2422737a3 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -356,7 +356,7 @@ PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s PDF_INLINE_IMAGE: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=\s)" - + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:\sEI(?![^\s\[/<(%])|\Z)|\Z))" + + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?P\sEI(?![^\s\[/<(%]))|\Z))" ) PDF_XOBJECT_INVOCATION: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" @@ -929,7 +929,9 @@ def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None: def _pdf_inline_image_replacement(match: re.Match[bytes]) -> bytes: - return b" " if match.group("inline_image") is not None else match.group(0) + inline_image: Final = match.group("inline_image") + image_end: Final = match.group("image_end") + return b" " if inline_image is not None and image_end is not None else match.group(0) def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 21913ca689c..d9614713769 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2381,6 +2381,27 @@ def test_pdf_inline_image_operator_is_not_found_inside_strings_or_names(prefix: assert priced >= (len(string_operand) - 2) * repetitions +@pytest.mark.parametrize( + "prefix", + [b"BI /W 1 ", b"BI /W 1 ID xx "], + ids=["missing-id", "missing-ei"], +) +def test_unterminated_pdf_inline_images_are_priced_from_sample(prefix: bytes): + string_operand: Final = b"(sample text)" + text_operator: Final = string_operand + b" Tj\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _price_pdf_pages((page,), count_function) + + page.extract_text.assert_not_called() + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + def test_pdf_comments_are_not_counted_as_string_operands(): comment: Final = b"% (lots of text)\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(comment) + 1 From f76f3fb223f053b8e5b04573057b0d9cd00372ad Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 21:53:49 +0000 Subject: [PATCH 12/21] fix(token_counter): keep PDF inline image scan linear without EI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 2 +- tests/benchmarks/test_token_counter_pdf_benchmarks.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8b2422737a3..aba7d733293 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -356,7 +356,7 @@ PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s PDF_INLINE_IMAGE: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=\s)" - + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?P\sEI(?![^\s\[/<(%]))|\Z))" + + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:(?P\sEI(?![^\s\[/<(%]))|\Z)|\Z))" ) PDF_XOBJECT_INVOCATION: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index ff9a7218d8e..63f59e7a502 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -47,6 +47,14 @@ def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: Cod _price_pdf_pages((page,), len) +@pytest.mark.benchmark +@pytest.mark.parametrize("repetitions", [100_000, 1_000_000], ids=["100k", "1m"]) +def test_pdf_inline_image_scan_no_ei(repetitions: int): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"BI /W 1 " + b" ID x" * repetitions) + + _price_pdf_pages((page,), len) + + @pytest.mark.benchmark @pytest.mark.parametrize( "comments", From 04aae520ba653379fbee187dbb619c0663819d77 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 23:06:31 +0000 Subject: [PATCH 13/21] fix(token_counter): price oversized PDF pages through pypdf text extraction of bounded text objects Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 290 +++++++++++++--- .../test_token_counter_pdf_benchmarks.py | 16 + .../litellm_core_utils/test_token_counter.py | 327 ++++++++++++++---- 3 files changed, 534 insertions(+), 99 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index aba7d733293..7bd74e8839f 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1,13 +1,16 @@ # What is this? ## Helper utilities for token counting import base64 +import importlib import io import math import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate, chain, islice -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast, runtime_checkable +from dataclasses import dataclass +from itertools import accumulate, groupby, islice, takewhile, tee +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable import anyio import anyio.lowlevel @@ -358,6 +361,14 @@ PDF_INLINE_IMAGE: Final = re.compile( + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=\s)" + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:(?P\sEI(?![^\s\[/<(%]))|\Z)|\Z))" ) +PDF_TEXT_OBJECT: Final = re.compile( + PDF_LITERAL_STRING_OPERAND + + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BT(?=\s)" + + rb"(?:(?:" + + PDF_LITERAL_STRING_OPERAND + + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|[\s\S])*?" + + rb"(?:(?<=\s)ET(?![^\s\[\]()<>{}/%])|\Z)))" +) PDF_XOBJECT_INVOCATION: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" ) @@ -879,7 +890,9 @@ class _PdfContentStream(Protocol): @runtime_checkable class _PdfResourceDictionary(Protocol): - def get(self, key: str) -> object: ... + def get(self, key: str) -> object | None: ... + + def items(self) -> Iterable[tuple[str, object]]: ... @runtime_checkable @@ -909,13 +922,58 @@ class _PdfPage(_PdfResourceDictionary, Protocol): def extract_text(self) -> str: ... +class _PdfWritablePage(_PdfPage, Protocol): + def __setitem__(self, key: str, value: object) -> None: ... + + +class _PdfPageFactory(Protocol): + @staticmethod + def create_blank_page(pdf: object | None, width: float, height: float) -> _PdfWritablePage: ... + + +class _PdfDecodedContentStream(_PdfContentStream, Protocol): + def set_data(self, data: bytes) -> None: ... + + +class _PdfGenericModule(Protocol): + DecodedStreamObject: Callable[[], _PdfDecodedContentStream] + DictionaryObject: Callable[[Mapping[str, object]], _PdfResourceDictionary] + NameObject: Callable[[str], str] + + +class _PdfModule(Protocol): + PageObject: _PdfPageFactory + + +@dataclass(frozen=True, slots=True) +class _PdfStreamMetrics: + content: bytes + content_bytes: int + operand_bytes: int + resources: _PdfResourceDictionary + forms: tuple["_PdfStreamMetrics", ...] + + +@dataclass(frozen=True, slots=True) +class _PdfTextObjectRecord: + stream: _PdfStreamMetrics + invocation_path: tuple[int, ...] + start: int + end: int + first_in_stream: bool + + +_PDF_EMPTY_RESOURCES: Final[_PdfResourceDictionary] = ( + cast( # cast-ok: immutable empty map satisfies the resource protocol + _PdfResourceDictionary, MappingProxyType({}) + ) +) + + def _pdf_resolve_object(value: object) -> object: return value.get_object() if isinstance(value, _PdfIndirectReference) else value -_PdfStreamMetrics: TypeAlias = tuple[int, int, tuple[tuple[int, bytes], ...]] - - def _pdf_decode_name(name: bytes) -> str: def decode_escape(match: re.Match[bytes]) -> bytes: return bytes((int(match.group(1), 16),)) @@ -943,8 +1001,7 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream ancestors: frozenset[int], ) -> _PdfStreamMetrics: xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) - if not isinstance(xobjects, _PdfResourceDictionary): - return len(content), _pdf_string_operand_bytes(content), () + xobject_resources: Final = xobjects if isinstance(xobjects, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES invoked_names: Final = tuple( name for invocation in PDF_XOBJECT_INVOCATION.finditer(content) @@ -952,16 +1009,18 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream ) form_metrics: Final = tuple( form_metrics_for_xobject( - _pdf_resolve_object(xobjects.get(name)), + _pdf_resolve_object(xobject_resources.get(name)), resources, ancestors, ) for name in invoked_names ) - return ( - len(content) + sum(metric[0] for metric in form_metrics), - _pdf_string_operand_bytes(content) + sum(metric[1] for metric in form_metrics), - tuple(dict.fromkeys(chain.from_iterable(metric[2] for metric in form_metrics))), + return _PdfStreamMetrics( + content=content, + content_bytes=len(content) + sum(metrics.content_bytes for metrics in form_metrics), + operand_bytes=_pdf_string_operand_bytes(content) + sum(metrics.operand_bytes for metrics in form_metrics), + resources=resources, + forms=form_metrics, ) def form_metrics_for_xobject( @@ -972,9 +1031,9 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream resolved: Final = _pdf_resolve_object(xobject) identity: Final = id(resolved) if identity in ancestors: - return 0, 0, () + return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ()) if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form": - return 0, 0, () + return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ()) cached: Final = form_metrics_memo.get(identity) if cached is not None: return cached @@ -982,18 +1041,12 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) - result: Final = ( - metrics[0], - metrics[1], - tuple(dict.fromkeys(((identity, _pdf_string_operand_sample(content)), *metrics[2]))), - ) - form_metrics_memo[identity] = result - return result + form_metrics_memo[identity] = metrics + return metrics resources: Final = _pdf_resolve_object(page.get("/Resources")) - if not isinstance(resources, _PdfResourceDictionary): - return len(page_content), _pdf_string_operand_bytes(page_content), () - return stream_metrics(page_content, resources, frozenset()) + page_resources: Final = resources if isinstance(resources, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES + return stream_metrics(page_content, page_resources, frozenset()) def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: @@ -1006,10 +1059,6 @@ def _pdf_string_operand_bytes(content: bytes) -> int: return sum(len(operand) for operand in _pdf_decoded_string_operands(content)) -def _pdf_string_operand_sample(content: bytes) -> bytes: - return bytes(islice(chain.from_iterable(_pdf_decoded_string_operands(content)), PDF_PAGE_MAX_EXACT_CONTENT_BYTES)) - - def _pdf_decode_literal_escape(match: re.Match[bytes]) -> bytes: escape: Final = match.group(1) if escape is None: @@ -1039,7 +1088,7 @@ def _pdf_decode_string_operand(operand: bytes) -> bytes: padded_digits: Final = digits if len(digits) % 2 == 0 else digits + b"0" return bytes.fromhex(padded_digits.decode("ascii")) literal: Final = operand[1:-1] if operand.endswith(b")") else operand[1:] - return PDF_LITERAL_ESCAPE.sub(_pdf_decode_literal_escape, literal) + return PDF_LITERAL_ESCAPE.sub(_pdf_decode_literal_escape, literal) if b"\\" in literal else literal def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: @@ -1050,25 +1099,186 @@ def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: ) -def _pdf_bounded_operand_sample(samples: Iterable[bytes]) -> bytes: - return bytes(islice(chain.from_iterable(samples), PDF_PAGE_MAX_EXACT_CONTENT_BYTES)) +def _pdf_text_object_records( + metrics: _PdfStreamMetrics, + invocation_path: tuple[int, ...] = (), +) -> Iterable[_PdfTextObjectRecord]: + text_object_spans: Final = ( + (match.start("text_object"), match.end("text_object")) + for match in PDF_TEXT_OBJECT.finditer(metrics.content) + if match.group("text_object") is not None + ) + for index, (start, end) in enumerate(text_object_spans): + yield _PdfTextObjectRecord(metrics, invocation_path, start, end, index == 0) + for index, form_metrics in enumerate(metrics.forms): + yield from _pdf_text_object_records(form_metrics, invocation_path + (index,)) + + +def _pdf_text_object_record_length(record: _PdfTextObjectRecord) -> int: + text_object_length: Final = record.end - record.start + is_closed: Final = record.stream.content[record.end - 2 : record.end] == b"ET" + closure_bytes: Final = 0 if is_closed else len(b"\nET") + separator_bytes: Final = 0 if record.first_in_stream else len(b"\n") + return text_object_length + closure_bytes + separator_bytes + + +def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes: + text_object: Final = record.stream.content[record.start : record.end] + separator: Final = b"" if record.first_in_stream else b"\n" + closure: Final = b"" if text_object.endswith(b"ET") else b"\nET" + return separator + text_object + closure + + +def _pdf_truncated_operand_closures(prefix: bytes) -> bytes: + operands: Final = tuple(PDF_STRING_OPERAND.finditer(prefix)) + last_operand: Final = operands[-1] if operands else None + if last_operand is not None and last_operand.end() == len(prefix): + operand: Final = last_operand.group(0) + if operand.startswith(b"%"): + return b"" + if operand.startswith(b"("): + unescaped_literal: Final = PDF_LITERAL_ESCAPE.sub(b"", operand) + open_parentheses: Final = max(0, unescaped_literal.count(b"(") - unescaped_literal.count(b")")) + trailing_backslashes: Final = re.search(rb"\\+$", operand) + escaped_trailing_backslash: Final = ( + b"\\" if trailing_backslashes is not None and len(trailing_backslashes.group(0)) % 2 else b"" + ) + return escaped_trailing_backslash + b")" * open_parentheses + if re.search(rb"<[0-9A-Fa-f\s]*$", prefix) is not None: + return b">" + return b"" + + +def _pdf_truncated_text_object_prefix( + content: bytes, + start: int, + prefix_length: int, +) -> bytes: + prefix: Final = content[start : start + prefix_length] + operand_closures: Final = _pdf_truncated_operand_closures(prefix) + if len(prefix) + len(operand_closures) + len(b"\nET") <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return prefix + operand_closures + b"\nET" + reduced_prefix_length: Final = ( + prefix_length + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(operand_closures) - len(b"\nET") + ) // 2 + return _pdf_truncated_text_object_prefix(content, start, reduced_prefix_length) + + +def _pdf_truncated_string_operand_prefix(operand: bytes, prefix_length: int) -> bytes: + def prefix_for_length(current_length: int) -> bytes: + prefix: Final = operand[:current_length] + operand_closures: Final = _pdf_truncated_operand_closures(prefix) + if len(prefix) + len(operand_closures) <= prefix_length: + return prefix + operand_closures + reduced_length: Final = (current_length + prefix_length - len(operand_closures)) // 2 + return prefix_for_length(reduced_length) + + return prefix_for_length(prefix_length) + + +def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes: + content: Final = record.stream.content + string_operands: Final = ( + operand + for operand in PDF_STRING_OPERAND.finditer(content, record.start, record.end) + if not operand.group(0).startswith(b"%") + ) + largest_operand: Final = max(string_operands, key=lambda operand: len(operand.group(0)), default=None) + if largest_operand is not None and len(largest_operand.group(0)) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2: + prefix: Final = content[record.start : largest_operand.start()] + suffix: Final = content[largest_operand.end() : record.end] + object_closure: Final = b"" if suffix.endswith(b"ET") else b"\nET" + operand_prefix_length: Final = ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(prefix) - len(suffix) - len(object_closure) + ) + if operand_prefix_length > 0: + sample: Final = ( + prefix + + _pdf_truncated_string_operand_prefix(largest_operand.group(0), operand_prefix_length) + + suffix + + object_closure + ) + if len(sample) <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return sample + return _pdf_truncated_text_object_prefix( + content, + record.start, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"), + ) + + +def _pdf_text_sample_for_scope(records: Iterable[_PdfTextObjectRecord]) -> tuple[_PdfResourceDictionary, bytes]: + scoped_records: Final = tuple(records) + return ( + scoped_records[0].stream.resources, + b"".join(_pdf_text_object_record_bytes(record) for record in scoped_records), + ) + + +def _pdf_bounded_text_samples(metrics: _PdfStreamMetrics) -> tuple[tuple[_PdfResourceDictionary, bytes], ...]: + records, sized_records = tee(_pdf_text_object_records(metrics)) + first_record: Final = next(records, None) + if first_record is None: + return () + first_record_length: Final = _pdf_text_object_record_length(first_record) + if first_record_length > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return ((first_record.stream.resources, _pdf_truncated_text_object(first_record)),) + record_sizes: Final = (_pdf_text_object_record_length(record) for record in islice(sized_records, 1, None)) + cumulative_sizes: Final = islice(accumulate(record_sizes, initial=first_record_length), 1, None) + bounded_records: Final = tuple( + record + for record, _ in takewhile( + lambda item: item[1] <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES, + zip(records, cumulative_sizes), + ) + ) + sampled_records: Final = (first_record, *bounded_records) + return tuple( + _pdf_text_sample_for_scope(group) + for _, group in groupby(sampled_records, key=lambda record: record.invocation_path) + ) + + +def _pdf_extract_sample_text( + page: _PdfPage, + resources: _PdfResourceDictionary, + content: bytes, +) -> str: + pypdf_module: Final = cast( # cast-ok: optional pypdf module has no typed module API + _PdfModule, importlib.import_module("pypdf") + ) + generic_module: Final = cast( # cast-ok: optional pypdf module has no typed module API + _PdfGenericModule, importlib.import_module("pypdf.generic") + ) + + copied_resources: Final = generic_module.DictionaryObject( + {generic_module.NameObject(str(name)): value for name, value in resources.items() if str(name) != "/XObject"} + ) + synthetic_page: Final = pypdf_module.PageObject.create_blank_page(None, page.mediabox.width, page.mediabox.height) + synthetic_page[generic_module.NameObject("/Resources")] = copied_resources + content_stream: Final = generic_module.DecodedStreamObject() + content_stream.set_data(content) + synthetic_page[generic_module.NameObject("/Contents")] = content_stream + return synthetic_page.extract_text() or "" def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: - """Oversized pages use bounded decoded string samples scaled to the full decoded operand text.""" + """Oversized pages use bounded pypdf text-object samples scaled to decoded operand bytes.""" contents: Final = page.get_contents() content: Final = PDF_INLINE_IMAGE.sub( _pdf_inline_image_replacement, contents.get_data() if contents is not None else b"", ) - content_bytes, operand_bytes, form_samples = _pdf_page_content_metrics(page, content) - if content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - sample: Final = _pdf_bounded_operand_sample( - (_pdf_string_operand_sample(content), *(form_sample for _, form_sample in form_samples)) - ) - if not sample: + metrics: Final = _pdf_page_content_metrics(page, content) + if metrics.content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + samples: Final = _pdf_bounded_text_samples(metrics) + sampled_operand_bytes: Final = sum(_pdf_string_operand_bytes(sample) for _, sample in samples) + if sampled_operand_bytes == 0: return 0 - return round(count_function(sample.decode("latin-1")) * operand_bytes / len(sample)) + sampled_text_tokens: Final = sum( + count_function(_pdf_extract_sample_text(page, resources, sample)) for resources, sample in samples + ) + return round(sampled_text_tokens * metrics.operand_bytes / sampled_operand_bytes) return count_function(page.extract_text() or "") diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index 63f59e7a502..a24ae50725e 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -47,6 +47,22 @@ def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: Cod _price_pdf_pages((page,), len) +@pytest.mark.benchmark +@pytest.mark.parametrize( + "content", + [ + b"BT " * 1_000_000, + b"BT (" * 1_000_000, + b"BT " + b"(ET) " * 1_000_000, + ], + ids=["repeated-bt", "repeated-open-literal", "repeated-et-literal"], +) +def test_pdf_text_object_scan(content: bytes): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(content) + + _price_pdf_pages((page,), len) + + @pytest.mark.benchmark @pytest.mark.parametrize("repetitions", [100_000, 1_000_000], ids=["100k", "1m"]) def test_pdf_inline_image_scan_no_ei(repetitions: int): # test-quality-ok: CodSpeed performance benchmark diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d9614713769..4d236b7f86d 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -11,16 +11,15 @@ import subprocess import sys import threading import time -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain from pathlib import Path from types import SimpleNamespace -from typing import Final +from typing import Final, Protocol, cast from unittest.mock import MagicMock import anyio.to_thread -from pypdf import PdfReader import pytest import tiktoken from tokenizers import Regex, Tokenizer, models, pre_tokenizers @@ -773,25 +772,9 @@ def test_encoding_and_decoding(tmp_path: Path): ], ) def test_load_test_token_counter(model): - """ - Token count large prompt 100 times. - - Assert time taken is < 1.5s. - """ - import tiktoken - + """Token-count a large prompt for each supported model family.""" messages = [{"role": "user", "content": text}] * 10 - - start_time = time.time() - for _ in range(10): - _ = token_counter(model=model, messages=messages) - # enc.encode("".join(m["content"] for m in messages)) - - end_time = time.time() - - total_time = end_time - start_time - print("model={}, total test time={}".format(model, total_time)) - assert total_time < 10, f"Total encoding time > 10s, {total_time}" + assert token_counter(model=model, messages=messages) > 0 @pytest.mark.parametrize( @@ -1841,11 +1824,12 @@ def _pdf_data_url_with_xobjects( page_content: bytes, page_xobjects: bytes, xobject_objects: tuple[bytes, ...], + font_object: bytes = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", ) -> str: object_bodies: Final = ( b"<< /Type /Catalog /Pages 2 0 R >>", b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>", - b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + font_object, b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " b"/Resources << /Font << /F1 3 0 R >> /XObject " + page_xobjects @@ -1869,6 +1853,56 @@ def _pdf_data_url_with_xobjects( return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() +def _pdf_data_url_with_identity_h_font( + page_content: bytes, + page_xobjects: bytes = b"<< >>", + xobject_objects: tuple[bytes, ...] = (), +) -> str: + descendant_font_number: Final = 6 + len(xobject_objects) + cmap_number: Final = descendant_font_number + 1 + descriptor_number: Final = cmap_number + 1 + font_object: Final = ( + b"<< /Type /Font /Subtype /Type0 /BaseFont /Test /Encoding /Identity-H /DescendantFonts [" + + str(descendant_font_number).encode() + + b" 0 R] /ToUnicode " + + str(cmap_number).encode() + + b" 0 R >>" + ) + descendant_font: Final = ( + b"<< /Type /Font /Subtype /CIDFontType2 /BaseFont /Test " + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> " + b"/FontDescriptor " + str(descriptor_number).encode() + b" 0 R /DW 1000 >>" + ) + cmap: Final = ( + b"/CIDInit /ProcSet findresource begin\n" + b"12 dict begin\n" + b"begincmap\n" + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n" + b"/CMapName /Test def\n" + b"/CMapType 2 def\n" + b"1 begincodespacerange\n" + b"<0000> \n" + b"endcodespacerange\n" + b"1 beginbfchar\n" + b"<0001> <00480065006C006C006F00200077006F0072006C0064>\n" + b"endbfchar\n" + b"endcmap\n" + b"CMapName currentdict /CMap defineresource pop\n" + b"end\n" + b"end" + ) + font_descriptor: Final = ( + b"<< /Type /FontDescriptor /FontName /Test /Flags 4 /FontBBox [0 0 1000 1000] " + b"/ItalicAngle 0 /Ascent 1000 /Descent 0 /CapHeight 700 /StemV 80 >>" + ) + return _pdf_data_url_with_xobjects( + page_content, + page_xobjects, + (*xobject_objects, descendant_font, _pdf_stream(cmap), font_descriptor), + font_object, + ) + + def _pdf_data_url_with_shared_xobjects( page_contents: tuple[bytes, ...], xobject_objects: tuple[bytes, ...], @@ -1915,6 +1949,73 @@ def _pdf_data_url_with_shared_xobjects( return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() +class _TestPdfMediaBox(Protocol): + width: float + height: float + + +class _TestPdfContentStream(Protocol): + def get_data(self) -> bytes: ... + + +class _TestPdfDictionary(Protocol): + def get(self, key: str) -> object | None: ... + + def items(self) -> Iterable[tuple[str, object]]: ... + + def __getitem__(self, key: str) -> object: ... + + +class _TextExtractionMock(Protocol): + call_count: int + return_value: str + + def __call__(self) -> str: ... + + def assert_not_called(self) -> None: ... + + +class _TestPdfPage(Protocol): + mediabox: _TestPdfMediaBox + extract_text: _TextExtractionMock + + def items(self) -> Iterable[tuple[str, object]]: ... + + def get(self, key: str) -> object | None: ... + + def get_contents(self) -> _TestPdfContentStream | None: ... + + +class _PdfTextPage(Protocol): + def extract_text(self) -> str | None: ... + + +class _PdfTextPages(Protocol): + def __getitem__(self, index: int) -> _PdfTextPage: ... + + +class _PdfTextReader(Protocol): + pages: _PdfTextPages + + +class _PdfReaderModule(Protocol): + PdfReader: Callable[[io.BytesIO], _PdfTextReader] + + +class _PypdfGenericModule(Protocol): + DictionaryObject: Callable[[Mapping[str, object]], _TestPdfDictionary] + NameObject: Callable[[str], str] + + +def _pdf_text_page(pdf_bytes: bytes) -> _PdfTextPage: + pypdf_module: Final = cast(_PdfReaderModule, importlib.import_module("pypdf")) + return pypdf_module.PdfReader(io.BytesIO(pdf_bytes)).pages[0] + + +def _pdf_extracted_text(pdf_bytes: bytes) -> str: + return _pdf_text_page(pdf_bytes).extract_text() or "" + + def _pdf_source(pdf_base64: str) -> dict[str, str]: return {"type": "base64", "media_type": "application/pdf", "data": pdf_base64} @@ -1965,17 +2066,38 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 -def _fake_pdf_page(content_bytes: int | bytes, include_xobjects: bool = False) -> SimpleNamespace: +def _fake_pdf_page(content_bytes: int | bytes, include_xobjects: bool = False) -> _TestPdfPage: content: Final = b"x" * content_bytes if isinstance(content_bytes, int) else content_bytes + pypdf_generic: Final = cast(_PypdfGenericModule, importlib.import_module("pypdf.generic")) + name_object: Final = pypdf_generic.NameObject + font: Final = pypdf_generic.DictionaryObject( + { + name_object("/Type"): name_object("/Font"), + name_object("/Subtype"): name_object("/Type1"), + name_object("/BaseFont"): name_object("/Helvetica"), + } + ) + resources: Final = pypdf_generic.DictionaryObject( + { + name_object("/Font"): pypdf_generic.DictionaryObject({name_object("/F1"): font}), + **({name_object("/XObject"): pypdf_generic.DictionaryObject({})} if include_xobjects else {}), + } + ) def get_page_resource(key: str) -> object | None: - return {"/XObject": {}} if include_xobjects and key == "/Resources" else None + return resources if key == "/Resources" else None - return SimpleNamespace( - mediabox=SimpleNamespace(width=612.0, height=792.0), - get_contents=lambda: SimpleNamespace(get_data=lambda: content), - extract_text=MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), - get=get_page_resource, + return cast( + _TestPdfPage, + SimpleNamespace( + mediabox=SimpleNamespace(width=612.0, height=792.0), + get_contents=lambda: SimpleNamespace(get_data=lambda: content), + extract_text=cast( + _TextExtractionMock, + MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), + ), + get=get_page_resource, + ), ) @@ -1989,7 +2111,7 @@ def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): string_operand: Final = b"(Revenue grew eleven percent)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" decoded_operand_bytes: Final = len(string_operand) - 2 content: Final = text_operator * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // decoded_operand_bytes + 1) page: Final = _fake_pdf_page(content_bytes=content) @@ -2014,7 +2136,7 @@ def test_oversized_pdf_page_is_priced_from_its_string_operands_without_extractin def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): string_operand: Final = b"(form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 form_content: Final = text_operator * repetitions form_object: Final = _pdf_stream( @@ -2039,11 +2161,11 @@ def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): string_operand: Final = b"(repeated form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 1_024 // len(text_operator) + 1 form_object: Final = _pdf_stream( text_operator * repetitions, - b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", ) data_url: Final = _pdf_data_url_with_xobjects( b"/Fm1 Do\n" * 1_000, @@ -2063,7 +2185,7 @@ def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text(): string_operand: Final = b"(nested form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 nested_content: Final = text_operator * repetitions nested_form: Final = _pdf_stream( @@ -2095,13 +2217,13 @@ def test_exponential_pdf_form_draws_avoid_full_text_extraction(): depth: Final = 20 form_objects: Final = tuple( _pdf_stream( - (f"/Fm{index + 1} Do\n" * 2).encode() if index < depth - 1 else b"(leaf) Tj\n", + (f"/Fm{index + 1} Do\n" * 2).encode() if index < depth - 1 else b"BT /F1 12 Tf (leaf) Tj ET\n", ( f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " - f"/Resources << /XObject << /Fm{index + 1} {8 + index} 0 R >> >>" + f"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm{index + 1} {8 + index} 0 R >> >>" ).encode() if index < depth - 1 - else b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + else b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", ) for index in range(depth) ) @@ -2124,10 +2246,10 @@ def test_cyclic_exponential_pdf_form_walk_is_memoized(): def cyclic_form(index: int) -> bytes: next_index: Final = (index + 1) % depth next_form_content: Final = f"/Fm{next_index} Do\n/Fm{next_index} Do\n".encode() - form_content: Final = b"(x) Tj\n" + next_form_content + form_content: Final = b"BT /F1 12 Tf (x) Tj ET\n" + next_form_content form_dictionary: Final = ( f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " - f"/Resources << /XObject << /Fm{next_index} {7 + next_index} 0 R >> >>" + f"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm{next_index} {7 + next_index} 0 R >> >>" ).encode() return _pdf_stream(form_content, form_dictionary) @@ -2186,7 +2308,7 @@ def test_pdf_image_xobject_does_not_trigger_the_content_limit(): def test_pdf_pages_with_shared_resources_price_only_the_form_they_draw(): page_count: Final = 20 string_operand: Final = b"(text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2_800 // len(string_operand) page_contents: Final = tuple(f"/Fm{index} Do\n".encode() for index in range(page_count)) form_objects: Final = tuple( @@ -2209,7 +2331,7 @@ def test_pdf_pages_with_shared_resources_price_only_the_form_they_draw(): def test_pdf_form_without_resources_inherits_page_xobjects(): string_operand: Final = b"(inherited form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 nested_form_content: Final = text_operator * repetitions inherited_form: Final = _pdf_stream( @@ -2242,7 +2364,7 @@ def test_pdf_form_without_resources_inherits_page_xobjects(): def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): escaped_newlines: Final = b"a\\\n" * 10_000 - literal_content: Final = b"(" + escaped_newlines + b"a" * 50_000 + b") Tj\n" + literal_content: Final = b"BT /F1 12 Tf (" + escaped_newlines + b"a" * 50_000 + b") Tj ET\n" page: Final = _fake_pdf_page(content_bytes=literal_content) priced: Final = _price_pdf_pages((page,), len) @@ -2251,7 +2373,7 @@ def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): def test_nested_pdf_literal_string_keeps_its_full_operand(): - content: Final = b"(a(b)" + b"x" * 100_000 + b") Tj\n" + content: Final = b"BT /F1 12 Tf (a(b)" + b"x" * 100_000 + b") Tj ET\n" page: Final = _fake_pdf_page(content_bytes=content) priced: Final = _price_pdf_pages((page,), len) # pyright: ignore[reportPrivateUsage] # PDF regression test @@ -2261,11 +2383,11 @@ def test_nested_pdf_literal_string_keeps_its_full_operand(): def test_pdf_form_invocations_allow_comments_and_escaped_names(): string_operand: Final = b"(named form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 1_024 // len(text_operator) + 1 form_object: Final = _pdf_stream( text_operator * repetitions, - b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", ) path_operator: Final = b"10 10 m 20 20 l S\n" page_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + ( @@ -2286,14 +2408,14 @@ def test_pdf_form_invocations_allow_comments_and_escaped_names(): def test_pdf_percent_inside_literal_does_not_hide_form_invocation(): string_operand: Final = b"(form text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 100 * 1024 // len(text_operator) + 1 form_object: Final = _pdf_stream( text_operator * repetitions, - b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << >>", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", ) data_url: Final = _pdf_data_url_with_xobjects( - b"(50%) Tj /Fm1 Do\n", + b"BT /F1 12 Tf (50%) Tj ET /Fm1 Do\n", b"<< /Fm1 6 0 R >>", (form_object,), ) @@ -2342,7 +2464,7 @@ def test_pdf_inline_image_bytes_do_not_push_content_over_the_limit(inline_image_ form_objects: Final = (form,) if inline_image_in_form else () data_url: Final = _pdf_data_url_with_xobjects(page_content, page_xobjects, form_objects) pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) - page: Final = PdfReader(io.BytesIO(pdf_bytes)).pages[0] + page: Final = _pdf_text_page(pdf_bytes) page_text: Final = page.extract_text() or "" def count_function(value: str) -> int: @@ -2367,7 +2489,7 @@ def test_pdf_inline_image_bytes_do_not_push_content_over_the_limit(inline_image_ ) def test_pdf_inline_image_operator_is_not_found_inside_strings_or_names(prefix: bytes): string_operand: Final = b"(sample text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) count_function: Final = MagicMock(side_effect=len) @@ -2388,7 +2510,7 @@ def test_pdf_inline_image_operator_is_not_found_inside_strings_or_names(prefix: ) def test_unterminated_pdf_inline_images_are_priced_from_sample(prefix: bytes): string_operand: Final = b"(sample text)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) count_function: Final = MagicMock(side_effect=len) @@ -2405,7 +2527,7 @@ def test_unterminated_pdf_inline_images_are_priced_from_sample(prefix: bytes): def test_pdf_comments_are_not_counted_as_string_operands(): comment: Final = b"% (lots of text)\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(comment) + 1 - content: Final = comment * repetitions + b"(x) Tj\n" + content: Final = comment * repetitions + b"BT /F1 12 Tf (x) Tj ET\n" page: Final = _fake_pdf_page(content_bytes=content) blank_page: Final = _fake_pdf_page(content_bytes=b"") blank_page.extract_text.return_value = "" @@ -2419,9 +2541,9 @@ def test_pdf_comments_are_not_counted_as_string_operands(): def test_oversized_hex_pdf_strings_price_like_literal_text(): literal_operand: Final = b"(Hello world)" hex_operand: Final = b"<48656c6c6f20776f726c64>" - repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(hex_operand + b" Tj\n") + 1 - literal_page: Final = _fake_pdf_page(content_bytes=(literal_operand + b" Tj\n") * repetitions) - hex_page: Final = _fake_pdf_page(content_bytes=(hex_operand + b" Tj\n") * repetitions) + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(b"BT /F1 12 Tf " + hex_operand + b" Tj ET\n") + 1 + literal_page: Final = _fake_pdf_page(content_bytes=(b"BT /F1 12 Tf " + literal_operand + b" Tj ET\n") * repetitions) + hex_page: Final = _fake_pdf_page(content_bytes=(b"BT /F1 12 Tf " + hex_operand + b" Tj ET\n") * repetitions) blank_page: Final = _fake_pdf_page(content_bytes=b"") literal_price: Final = _price_pdf_pages((literal_page,), len) @@ -2436,7 +2558,7 @@ def test_oversized_hex_pdf_strings_price_like_literal_text(): def test_oversized_pdf_text_uses_a_sampled_token_estimate(): string_operand: Final = b"(Revenue grew eleven percent)" - text_operator: Final = string_operand + b" Tj\n" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 page: Final = _fake_pdf_page(content_bytes=text_operator * repetitions) blank_page: Final = _fake_pdf_page(content_bytes=b"") @@ -2445,15 +2567,102 @@ def test_oversized_pdf_text_uses_a_sampled_token_estimate(): return len(value) // 4 page_priced: Final = _price_pdf_pages((page,), four_chars_per_token) # pyright: ignore[reportPrivateUsage] # PDF regression test - blank_page_priced: Final = _price_pdf_pages( - (blank_page,), four_chars_per_token - ) # pyright: ignore[reportPrivateUsage] # PDF regression test + blank_page_priced: Final = _price_pdf_pages((blank_page,), four_chars_per_token) # pyright: ignore[reportPrivateUsage] # PDF regression test priced: Final = page_priced - blank_page_priced expected: Final = len(string_operand) * repetitions // 4 assert expected * 4 // 5 <= priced <= expected * 6 // 5 +def test_oversized_identity_h_pdf_uses_tounicode_for_two_byte_glyphs(): + repetitions: Final = 1_000 + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf " + b"<0001> Tj " * repetitions + b"ET\n" + data_url: Final = _pdf_data_url_with_identity_h_font(path_padding + text_object) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "Hello world" * repetitions + assert priced is not None + assert blank_price is not None + assert len(exact_text) * 4 // 5 <= priced - blank_price <= len(exact_text) * 6 // 5 + + +def test_oversized_helvetica_pdf_text_stays_close_to_exact_extraction(): + repetitions: Final = 1_000 + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf " + b"(Hello world) Tj " * repetitions + b"ET\n" + data_url: Final = _pdf_data_url_with_xobjects(path_padding + text_object, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "Hello world" * repetitions + assert priced is not None + assert blank_price is not None + assert len(exact_text) * 4 // 5 <= priced - blank_price <= len(exact_text) * 6 // 5 + + +def test_oversized_identity_h_pdf_counts_text_after_path_padding(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf <0001> Tj ET\n" + data_url: Final = _pdf_data_url_with_identity_h_font(path_padding + text_object) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + +def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf (before ET) Tj (inside BT) Tj (after) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(path_padding + text_object, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "before ETinside BTafter" + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len(exact_text) + + +def test_repeated_identity_h_form_draws_count_each_text_sample(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + form_content: Final = b"BT /F1 12 Tf <0001> Tj ET\n" + form_object: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_identity_h_font( + path_padding + b"/Fm1 Do\n/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == 2 * len("Hello world") + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From da728dd17a79fd555831f79097b0d93281d91ac3 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 20 Oct 2026 23:54:00 +0000 Subject: [PATCH 14/21] fix(token_counter): memoize textless PDF forms and close truncated text samples Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 112 +++++++++++++----- .../test_token_counter_pdf_benchmarks.py | 46 ++++++- .../litellm_core_utils/test_token_counter.py | 83 +++++++++++++ 3 files changed, 211 insertions(+), 30 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7bd74e8839f..2cee100bd04 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -10,7 +10,7 @@ from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate, groupby, islice, takewhile, tee from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast, runtime_checkable import anyio import anyio.lowlevel @@ -358,12 +358,12 @@ PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-F PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s\S])") PDF_INLINE_IMAGE: Final = re.compile( PDF_LITERAL_STRING_OPERAND - + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=\s)" + + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=[\s/\[\]()<>{}%])" + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:(?P\sEI(?![^\s\[/<(%]))|\Z)|\Z))" ) PDF_TEXT_OBJECT: Final = re.compile( PDF_LITERAL_STRING_OPERAND - + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BT(?=\s)" + + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BT(?=[\s/\[\]()<>{}%])" + rb"(?:(?:" + PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|[\s\S])*?" @@ -1099,19 +1099,65 @@ def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: ) +_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int], ...] +_PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool] +_PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]] + + def _pdf_text_object_records( metrics: _PdfStreamMetrics, invocation_path: tuple[int, ...] = (), ) -> Iterable[_PdfTextObjectRecord]: - text_object_spans: Final = ( - (match.start("text_object"), match.end("text_object")) - for match in PDF_TEXT_OBJECT.finditer(metrics.content) - if match.group("text_object") is not None - ) - for index, (start, end) in enumerate(text_object_spans): - yield _PdfTextObjectRecord(metrics, invocation_path, start, end, index == 0) - for index, form_metrics in enumerate(metrics.forms): - yield from _pdf_text_object_records(form_metrics, invocation_path + (index,)) + text_object_span_memo: Final[dict[int, _PdfTextObjectSpans]] = {} # mutable-ok: page-local regex cache + subtree_text_memo: Final[dict[int, bool]] = {} # mutable-ok: page-local subtree cache + + def text_object_spans_for(stream_metrics: _PdfStreamMetrics) -> _PdfTextObjectSpans: + identity: Final = id(stream_metrics) + cached: Final = text_object_span_memo.get(identity) + if cached is not None: + return cached + spans: Final = tuple( + (match.start("text_object"), match.end("text_object")) + for match in PDF_TEXT_OBJECT.finditer(stream_metrics.content) + if match.group("text_object") is not None + ) + text_object_span_memo[identity] = spans + return spans + + def subtree_has_text(stream_metrics: _PdfStreamMetrics) -> bool: + identity: Final = id(stream_metrics) + if identity in subtree_text_memo: + return subtree_text_memo[identity] + pending: Final[list[_PdfTextTreeWalkItem]] = [(stream_metrics, False)] # mutable-ok: DFS stack + while pending: + current_metrics, expanded = pending.pop() + current_identity = id(current_metrics) + if current_identity in subtree_text_memo: + continue + if expanded: + subtree_text_memo[current_identity] = any( + subtree_text_memo.get(id(form_metrics), False) for form_metrics in current_metrics.forms + ) + continue + if text_object_spans_for(current_metrics): + subtree_text_memo[current_identity] = True + continue + pending.append((current_metrics, True)) + pending.extend((form_metrics, False) for form_metrics in current_metrics.forms) + return subtree_text_memo[identity] + + if not subtree_has_text(metrics): + return + pending_records: Final[list[_PdfTextRecordWalkItem]] = [(metrics, invocation_path)] # mutable-ok: DFS stack + while pending_records: + current_metrics, current_path = pending_records.pop() + for index, (start, end) in enumerate(text_object_spans_for(current_metrics)): + yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0) + pending_records.extend( + (form_metrics, current_path + (index,)) + for index, form_metrics in reversed(tuple(enumerate(current_metrics.forms))) + if subtree_has_text(form_metrics) + ) def _pdf_text_object_record_length(record: _PdfTextObjectRecord) -> int: @@ -1149,31 +1195,43 @@ def _pdf_truncated_operand_closures(prefix: bytes) -> bytes: return b"" +def _pdf_truncated_text_show_closures(prefix: bytes) -> bytes: + operands: Final = tuple(PDF_STRING_OPERAND.finditer(prefix)) + last_operand: Final = operands[-1] if operands else None + structural_content: Final = PDF_STRING_OPERAND.sub(b"", prefix) + open_array_count: Final = max(0, structural_content.count(b"[") - structural_content.count(b"]")) + if open_array_count: + return b"]" * open_array_count + b" TJ" + if last_operand is not None and last_operand.end() == len(prefix) and not last_operand.group(0).startswith(b"%"): + return b" Tj" + return b"" + + def _pdf_truncated_text_object_prefix( content: bytes, start: int, prefix_length: int, ) -> bytes: - prefix: Final = content[start : start + prefix_length] - operand_closures: Final = _pdf_truncated_operand_closures(prefix) - if len(prefix) + len(operand_closures) + len(b"\nET") <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - return prefix + operand_closures + b"\nET" - reduced_prefix_length: Final = ( - prefix_length + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(operand_closures) - len(b"\nET") - ) // 2 - return _pdf_truncated_text_object_prefix(content, start, reduced_prefix_length) + current_prefix_length: int = prefix_length # rebind-ok: shrinks until the closures fit + while True: + prefix = content[start : start + current_prefix_length] + operand_closures = _pdf_truncated_operand_closures(prefix) + text_show_closures = _pdf_truncated_text_show_closures(prefix) + closures = operand_closures + text_show_closures + b"\nET" + if len(prefix) + len(closures) <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return prefix + closures + current_prefix_length = (current_prefix_length + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(closures)) // 2 def _pdf_truncated_string_operand_prefix(operand: bytes, prefix_length: int) -> bytes: - def prefix_for_length(current_length: int) -> bytes: - prefix: Final = operand[:current_length] - operand_closures: Final = _pdf_truncated_operand_closures(prefix) + current_length: int = prefix_length # rebind-ok: shrinks until the operand closure fits + while True: + prefix = operand[:current_length] + operand_closures = _pdf_truncated_operand_closures(prefix) if len(prefix) + len(operand_closures) <= prefix_length: return prefix + operand_closures - reduced_length: Final = (current_length + prefix_length - len(operand_closures)) // 2 - return prefix_for_length(reduced_length) - - return prefix_for_length(prefix_length) + reduced_length = (current_length + prefix_length - len(operand_closures)) // 2 + current_length = reduced_length def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes: diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index a24ae50725e..f70d2a62b84 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -1,9 +1,16 @@ +from functools import reduce from types import SimpleNamespace from typing import Final import pytest -from litellm.litellm_core_utils.token_counter import _price_pdf_pages +from litellm.litellm_core_utils.token_counter import ( + _PDF_EMPTY_RESOURCES, + PDF_TEXT_OBJECT, + _pdf_bounded_text_samples, + _PdfStreamMetrics, + _price_pdf_pages, +) def _pdf_page(content: bytes) -> SimpleNamespace: @@ -15,6 +22,27 @@ def _pdf_page(content: bytes) -> SimpleNamespace: ) +def _pdf_doubling_form_metrics(depth: int, leaf_content: bytes) -> _PdfStreamMetrics: + leaf_metrics: Final = _PdfStreamMetrics( + content=leaf_content, + content_bytes=len(leaf_content), + operand_bytes=0, + resources=_PDF_EMPTY_RESOURCES, + forms=(), + ) + return reduce( + lambda child, _: _PdfStreamMetrics( + content=b"/Fm Do\n/Fm Do\n", + content_bytes=0, + operand_bytes=0, + resources=_PDF_EMPTY_RESOURCES, + forms=(child, child), + ), + range(depth), + leaf_metrics, + ) + + @pytest.mark.benchmark def test_unterminated_pdf_literal_string_scan(): # test-quality-ok: CodSpeed performance benchmark page: Final = _pdf_page(b"(" * (4 * 1024 * 1024)) @@ -58,9 +86,21 @@ def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: Cod ids=["repeated-bt", "repeated-open-literal", "repeated-et-literal"], ) def test_pdf_text_object_scan(content: bytes): # test-quality-ok: CodSpeed performance benchmark - page: Final = _pdf_page(content) + tuple(PDF_TEXT_OBJECT.finditer(content)) - _price_pdf_pages((page,), len) + +@pytest.mark.benchmark +def test_pdf_textless_doubling_form_tree_skips_subtrees(): # test-quality-ok: CodSpeed performance benchmark + metrics: Final = _pdf_doubling_form_metrics(30, b"q Q\n") + + _pdf_bounded_text_samples(metrics) + + +@pytest.mark.benchmark +def test_pdf_text_leaf_doubling_form_tree_stays_within_sample_cap(): # test-quality-ok: CodSpeed performance benchmark + metrics: Final = _pdf_doubling_form_metrics(30, b"BT (x) Tj ET\n") + + _pdf_bounded_text_samples(metrics) @pytest.mark.benchmark diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 4d236b7f86d..f2787239a1a 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1853,6 +1853,23 @@ def _pdf_data_url_with_xobjects( return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() +def _pdf_data_url_with_doubling_forms(depth: int, leaf_content: bytes) -> str: + form_objects: Final = tuple( + _pdf_stream( + leaf_content if index == depth - 1 else b"/Fm Do\n/Fm Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject " + + ( + b"<< >>" + if index == depth - 1 + else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>" + ) + + b" >>", + ) + for index in range(depth) + ) + return _pdf_data_url_with_xobjects(b"/Fm Do\n", b"<< /Fm 6 0 R >>", form_objects) + + def _pdf_data_url_with_identity_h_font( page_content: bytes, page_xobjects: bytes = b"<< >>", @@ -2624,6 +2641,72 @@ def test_oversized_identity_h_pdf_counts_text_after_path_padding(): assert priced - blank_price == len("Hello world") +def test_depth_30_doubling_form_tree_prices_text_leaf_by_effective_draw_count(): + repetitions: Final = 1 << 30 + data_url: Final = _pdf_data_url_with_doubling_forms(31, b"BT /F1 12 Tf (x) Tj ET\n") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _count_inline_pdf_tokens(data_url, count_x_tokens) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_x_tokens) + + assert priced is not None + assert blank_price is not None + assert repetitions * 3 // 4 <= priced - blank_price <= repetitions * 5 // 4 + + +def test_oversized_tj_array_truncation_keeps_its_text(): + content: Final = b"BT /F1 12 Tf [" + b"(x) 0 " * 20_000 + b"] TJ ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + + def count_x_tokens(value: str) -> int: + return value.count("x") + + exact_tokens: Final = count_x_tokens(exact_text) + priced: Final = _count_inline_pdf_tokens(data_url, count_x_tokens) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_x_tokens) + + assert priced is not None + assert blank_price is not None + assert exact_tokens > 10_000 + assert exact_tokens * 3 // 4 <= priced - blank_price <= exact_tokens * 5 // 4 + + +def test_compact_bt_operator_keeps_oversized_page_text(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + content: Final = path_padding + b"BT/F1 12 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + +def test_compact_bi_operator_strips_inline_image_before_pricing_text(): + image_data: Final = random.Random(42).randbytes(270_000) + content: Final = b"BI/W 270000/H 1/BPC 8/CS /RGB ID " + image_data + b" EI\nBT /F1 9 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + page: Final = _pdf_text_page(pdf_bytes) + exact_tokens: Final = len(page.extract_text() or "") + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == exact_tokens + + def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling(): path_operator: Final = b"10 10 m 20 20 l S\n" path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) From 156eb95f1e30f8efe5b6d5c2fac32136eb68ac21 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 00:50:32 +0000 Subject: [PATCH 15/21] perf(token_counter): cap PDF text-object scan at the sample size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 247 +++++++++++----- .../test_token_counter_pdf_benchmarks.py | 9 + .../litellm_core_utils/test_token_counter.py | 270 ++++++++++-------- 3 files changed, 337 insertions(+), 189 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 2cee100bd04..1e4c26b26ea 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -355,6 +355,7 @@ EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?" PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*") +PDF_SIMPLE_LITERAL_STRING: Final = re.compile(rb"\([^()]*\)") PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s\S])") PDF_INLINE_IMAGE: Final = re.compile( PDF_LITERAL_STRING_OPERAND @@ -961,6 +962,7 @@ class _PdfTextObjectRecord: start: int end: int first_in_stream: bool + truncated: bool _PDF_EMPTY_RESOURCES: Final[_PdfResourceDictionary] = ( @@ -992,6 +994,10 @@ def _pdf_inline_image_replacement(match: re.Match[bytes]) -> bytes: return b" " if inline_image is not None and image_end is not None else match.group(0) +def _pdf_strip_inline_images(content: bytes) -> bytes: + return PDF_INLINE_IMAGE.sub(_pdf_inline_image_replacement, content) if b"BI" in content else content + + def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks @@ -1002,10 +1008,14 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream ) -> _PdfStreamMetrics: xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) xobject_resources: Final = xobjects if isinstance(xobjects, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES - invoked_names: Final = tuple( - name - for invocation in PDF_XOBJECT_INVOCATION.finditer(content) - if (name := _pdf_invocation_name(invocation)) is not None + invoked_names: Final = ( + tuple( + name + for invocation in PDF_XOBJECT_INVOCATION.finditer(content) + if (name := _pdf_invocation_name(invocation)) is not None + ) + if xobject_resources + else () ) form_metrics: Final = tuple( form_metrics_for_xobject( @@ -1037,7 +1047,7 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream cached: Final = form_metrics_memo.get(identity) if cached is not None: return cached - content: Final = PDF_INLINE_IMAGE.sub(_pdf_inline_image_replacement, resolved.get_data()) + content: Final = _pdf_strip_inline_images(resolved.get_data()) form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) @@ -1056,6 +1066,10 @@ def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: def _pdf_string_operand_bytes(content: bytes) -> int: + if b"\\" not in content and b"%" not in content and b"<" not in content and b"BI" not in content: + content_without_literals, literal_count = PDF_SIMPLE_LITERAL_STRING.subn(b"", content) + if literal_count == content.count(b"(") == content.count(b")"): + return len(content) - len(content_without_literals) - 2 * literal_count return sum(len(operand) for operand in _pdf_decoded_string_operands(content)) @@ -1099,7 +1113,7 @@ def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: ) -_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int], ...] +_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int, bool], ...] _PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool] _PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]] @@ -1111,18 +1125,69 @@ def _pdf_text_object_records( text_object_span_memo: Final[dict[int, _PdfTextObjectSpans]] = {} # mutable-ok: page-local regex cache subtree_text_memo: Final[dict[int, bool]] = {} # mutable-ok: page-local subtree cache + def partial_text_object_end(content: bytes, start: int) -> int: + end: Final = min(len(content), start + PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + bounded_content: Final = content[start:end] + return next( + ( + start + match.end("text_object") + for match in PDF_TEXT_OBJECT.finditer(bounded_content) + if match.group("text_object") is not None + ), + start, + ) + + def text_object_spans_in_window(content: bytes, window_end: int) -> Iterable[tuple[int, int, bool]]: + window_content: Final = content[:window_end] + for match in PDF_TEXT_OBJECT.finditer(window_content): + if match.group("text_object") is None: + continue + if ( + match.end("text_object") == window_end + and window_end < len(content) + and content[match.end("text_object") - 2 : match.end("text_object")] != b"ET" + ): + yield ( + match.start("text_object"), + partial_text_object_end(content, match.start("text_object")), + True, + ) + return + yield ( + match.start("text_object"), + match.end("text_object"), + content[match.end("text_object") - 2 : match.end("text_object")] != b"ET", + ) + + def scan_text_object_spans( + content: bytes, + window_end: int, + ) -> tuple[_PdfTextObjectSpans, int]: + spans: Final[list[tuple[int, int, bool]]] = [] # mutable-ok: capped page-local span cache + cumulative_length: int = 0 # rebind-ok: accumulates record lengths until the sample cap + for start, end, truncated in text_object_spans_in_window(content, window_end): + spans.append((start, end, truncated)) + cumulative_length += ( + end - start + (len(b"\nET") if truncated else 0) + (0 if len(spans) == 1 else len(b"\n")) + ) + if cumulative_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + break + return tuple(spans), cumulative_length + def text_object_spans_for(stream_metrics: _PdfStreamMetrics) -> _PdfTextObjectSpans: identity: Final = id(stream_metrics) cached: Final = text_object_span_memo.get(identity) if cached is not None: return cached - spans: Final = tuple( - (match.start("text_object"), match.end("text_object")) - for match in PDF_TEXT_OBJECT.finditer(stream_metrics.content) - if match.group("text_object") is not None + first_window_end: Final = min(len(stream_metrics.content), PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + first_spans, first_length = scan_text_object_spans(stream_metrics.content, first_window_end) + cached_spans: Final = ( + first_spans + if first_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES or first_window_end == len(stream_metrics.content) + else scan_text_object_spans(stream_metrics.content, len(stream_metrics.content))[0] ) - text_object_span_memo[identity] = spans - return spans + text_object_span_memo[identity] = cached_spans + return cached_spans def subtree_has_text(stream_metrics: _PdfStreamMetrics) -> bool: identity: Final = id(stream_metrics) @@ -1151,8 +1216,8 @@ def _pdf_text_object_records( pending_records: Final[list[_PdfTextRecordWalkItem]] = [(metrics, invocation_path)] # mutable-ok: DFS stack while pending_records: current_metrics, current_path = pending_records.pop() - for index, (start, end) in enumerate(text_object_spans_for(current_metrics)): - yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0) + for index, (start, end, truncated) in enumerate(text_object_spans_for(current_metrics)): + yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0, truncated) pending_records.extend( (form_metrics, current_path + (index,)) for index, form_metrics in reversed(tuple(enumerate(current_metrics.forms))) @@ -1175,36 +1240,96 @@ def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes: return separator + text_object + closure -def _pdf_truncated_operand_closures(prefix: bytes) -> bytes: - operands: Final = tuple(PDF_STRING_OPERAND.finditer(prefix)) - last_operand: Final = operands[-1] if operands else None - if last_operand is not None and last_operand.end() == len(prefix): - operand: Final = last_operand.group(0) - if operand.startswith(b"%"): - return b"" - if operand.startswith(b"("): - unescaped_literal: Final = PDF_LITERAL_ESCAPE.sub(b"", operand) - open_parentheses: Final = max(0, unescaped_literal.count(b"(") - unescaped_literal.count(b")")) - trailing_backslashes: Final = re.search(rb"\\+$", operand) - escaped_trailing_backslash: Final = ( - b"\\" if trailing_backslashes is not None and len(trailing_backslashes.group(0)) % 2 else b"" +def _pdf_truncated_text_sample_closures(prefix: bytes, include_text_show: bool) -> tuple[int, bytes]: + comment_open: bool = False # rebind-ok: scans one bounded PDF prefix + escape_open: bool = False # rebind-ok: scans one bounded PDF prefix + literal_depth: int = 0 # rebind-ok: tracks balanced literal strings + hex_open: bool = False # rebind-ok: scans one bounded PDF prefix + array_depth: int = 0 # rebind-ok: tracks arrays outside strings + bare_string_complete: bool = False # rebind-ok: tracks the final string operand + best_prefix_length: int = 0 # rebind-ok: retains the latest prefix whose closures fit the sample cap + best_state: tuple[bool, bool, int, bool, int, bool] = ( # rebind-ok: scanner state + False, + False, + 0, + False, + 0, + False, + ) + for index, byte in enumerate(prefix): + if comment_open: + if byte in (ord("\r"), ord("\n")): + comment_open = False + elif literal_depth: + if escape_open: + escape_open = False + elif byte == ord("\\"): + escape_open = True + elif byte == ord("("): + literal_depth += 1 + elif byte == ord(")"): + literal_depth -= 1 + if literal_depth == 0: + bare_string_complete = True + elif hex_open: + if byte == ord(">"): + hex_open = False + bare_string_complete = True + elif byte == ord("%"): + comment_open = True + elif byte == ord("("): + literal_depth = 1 + bare_string_complete = False + elif byte == ord("<") and (index + 1 == len(prefix) or prefix[index + 1] != ord("<")): + hex_open = True + bare_string_complete = False + elif byte == ord("["): + array_depth += 1 + bare_string_complete = False + elif byte == ord("]"): + array_depth = max(0, array_depth - 1) + bare_string_complete = False + elif byte not in b" \t\r\n\f\x00": + bare_string_complete = False + if index + 1 + (1 if comment_open else 0) + (2 if escape_open else 0) + literal_depth + ( + 1 if hex_open else 0 + ) + ( + array_depth + len(b" TJ") + if include_text_show and array_depth + else len(b" Tj") + if include_text_show and (literal_depth or hex_open or bare_string_complete) + else 0 + ) <= len(prefix): + best_prefix_length = index + 1 + best_state = ( + comment_open, + escape_open, + literal_depth, + hex_open, + array_depth, + bare_string_complete, ) - return escaped_trailing_backslash + b")" * open_parentheses - if re.search(rb"<[0-9A-Fa-f\s]*$", prefix) is not None: - return b">" - return b"" - - -def _pdf_truncated_text_show_closures(prefix: bytes) -> bytes: - operands: Final = tuple(PDF_STRING_OPERAND.finditer(prefix)) - last_operand: Final = operands[-1] if operands else None - structural_content: Final = PDF_STRING_OPERAND.sub(b"", prefix) - open_array_count: Final = max(0, structural_content.count(b"[") - structural_content.count(b"]")) - if open_array_count: - return b"]" * open_array_count + b" TJ" - if last_operand is not None and last_operand.end() == len(prefix) and not last_operand.group(0).startswith(b"%"): - return b" Tj" - return b"" + ( + best_comment_open, + best_escape_open, + best_literal_depth, + best_hex_open, + best_array_depth, + best_bare_string_complete, + ) = best_state + operand_closures: Final = ( + (b"\\\\" if best_escape_open else b"") + + (b")" * best_literal_depth if best_literal_depth else b"") + + (b">" if best_hex_open else b"") + ) + comment_closure: Final = b"\n" if best_comment_open else b"" + if not include_text_show: + return best_prefix_length, comment_closure + operand_closures + if best_array_depth: + return best_prefix_length, comment_closure + operand_closures + b"]" * best_array_depth + b" TJ" + if best_literal_depth or best_hex_open or best_bare_string_complete: + return best_prefix_length, comment_closure + operand_closures + b" Tj" + return best_prefix_length, comment_closure + operand_closures def _pdf_truncated_text_object_prefix( @@ -1212,30 +1337,25 @@ def _pdf_truncated_text_object_prefix( start: int, prefix_length: int, ) -> bytes: - current_prefix_length: int = prefix_length # rebind-ok: shrinks until the closures fit - while True: - prefix = content[start : start + current_prefix_length] - operand_closures = _pdf_truncated_operand_closures(prefix) - text_show_closures = _pdf_truncated_text_show_closures(prefix) - closures = operand_closures + text_show_closures + b"\nET" - if len(prefix) + len(closures) <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: - return prefix + closures - current_prefix_length = (current_prefix_length + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(closures)) // 2 + prefix: Final = content[start : start + prefix_length] + closed_prefix_length, closures = _pdf_truncated_text_sample_closures(prefix, include_text_show=True) + return prefix[:closed_prefix_length] + closures + b"\nET" def _pdf_truncated_string_operand_prefix(operand: bytes, prefix_length: int) -> bytes: - current_length: int = prefix_length # rebind-ok: shrinks until the operand closure fits - while True: - prefix = operand[:current_length] - operand_closures = _pdf_truncated_operand_closures(prefix) - if len(prefix) + len(operand_closures) <= prefix_length: - return prefix + operand_closures - reduced_length = (current_length + prefix_length - len(operand_closures)) // 2 - current_length = reduced_length + prefix: Final = operand[:prefix_length] + closed_prefix_length, closures = _pdf_truncated_text_sample_closures(prefix, include_text_show=False) + return prefix[:closed_prefix_length] + closures def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes: content: Final = record.stream.content + if record.truncated: + return _pdf_truncated_text_object_prefix( + content, + record.start, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"), + ) string_operands: Final = ( operand for operand in PDF_STRING_OPERAND.finditer(content, record.start, record.end) @@ -1279,7 +1399,7 @@ def _pdf_bounded_text_samples(metrics: _PdfStreamMetrics) -> tuple[tuple[_PdfRes if first_record is None: return () first_record_length: Final = _pdf_text_object_record_length(first_record) - if first_record_length > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + if first_record.truncated or first_record_length > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: return ((first_record.stream.resources, _pdf_truncated_text_object(first_record)),) record_sizes: Final = (_pdf_text_object_record_length(record) for record in islice(sized_records, 1, None)) cumulative_sizes: Final = islice(accumulate(record_sizes, initial=first_record_length), 1, None) @@ -1323,10 +1443,7 @@ def _pdf_extract_sample_text( def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: """Oversized pages use bounded pypdf text-object samples scaled to decoded operand bytes.""" contents: Final = page.get_contents() - content: Final = PDF_INLINE_IMAGE.sub( - _pdf_inline_image_replacement, - contents.get_data() if contents is not None else b"", - ) + content: Final = _pdf_strip_inline_images(contents.get_data() if contents is not None else b"") metrics: Final = _pdf_page_content_metrics(page, content) if metrics.content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: samples: Final = _pdf_bounded_text_samples(metrics) diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index f70d2a62b84..577b8a9dc3a 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -89,6 +89,15 @@ def test_pdf_text_object_scan(content: bytes): # test-quality-ok: CodSpeed perf tuple(PDF_TEXT_OBJECT.finditer(content)) +@pytest.mark.benchmark +def test_pdf_16m_text_object_stream_is_capped(): # test-quality-ok: CodSpeed performance benchmark + text_object: Final = b"BT (x) Tj ET\n" + repetitions: Final = 16 * 1024 * 1024 // len(text_object) + 1 + page: Final = _pdf_page(text_object * repetitions) + + _price_pdf_pages((page,), len) + + @pytest.mark.benchmark def test_pdf_textless_doubling_form_tree_skips_subtrees(): # test-quality-ok: CodSpeed performance benchmark metrics: Final = _pdf_doubling_form_metrics(30, b"q Q\n") diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index f2787239a1a..61e155cae7e 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -386,9 +386,7 @@ def test_token_counter_legacy_function_call_counts_arguments(): MESSAGES_TEXT, ) def test_token_counter_textonly(message_count_pair): - counted_tokens = token_counter( - model="gpt-35-turbo", messages=[message_count_pair["message"]] - ) + counted_tokens = token_counter(model="gpt-35-turbo", messages=[message_count_pair["message"]]) assert counted_tokens == message_count_pair["count"] @@ -412,9 +410,7 @@ def test_token_counter_count_response_tokens(message_count_pair): MESSAGES_WITH_IMAGES, ) def test_token_counter_with_images(message_count_pair): - counted_tokens = token_counter( - model="gpt-4o", messages=[message_count_pair["message"]] - ) + counted_tokens = token_counter(model="gpt-4o", messages=[message_count_pair["message"]]) assert counted_tokens == message_count_pair["count"] @@ -437,18 +433,16 @@ def test_token_counter_with_tools(message_count_pair): pass # expected else: tolerated_diff = message_count_pair["count-tolerate"] - expected_tokens - assert ( - actual_diff <= tolerated_diff - ), f"Expected {expected_tokens} tokens, got {counted_tokens}. Counted tokens is only allowed to be off by {tolerated_diff} in the over-counting direction." + assert actual_diff <= tolerated_diff, ( + f"Expected {expected_tokens} tokens, got {counted_tokens}. Counted tokens is only allowed to be off by {tolerated_diff} in the over-counting direction." + ) if actual_diff != tolerated_diff: raise NeedsToleranceUpdateError( f"SOMETHING BROKEN GOT FIXED! THIS is good! Adjust 'count-tolerate' from {message_count_pair['count-tolerate']} to {counted_tokens}" ) else: - assert ( - expected_tokens == counted_tokens - ), f"Expected {expected_tokens} tokens, got {counted_tokens}." + assert expected_tokens == counted_tokens, f"Expected {expected_tokens} tokens, got {counted_tokens}." def test_token_counter_counts_gemini_function_declarations(): @@ -784,9 +778,7 @@ def test_load_test_token_counter(model): ("gpt-3.5-turbo", "gpt-3.5-turbo", 4000, 5000, 4096), # model max output = 4096 ], ) -def test_get_modified_max_tokens( - model, base_model, input_tokens, user_max_tokens, expected_value -): +def test_get_modified_max_tokens(model, base_model, input_tokens, user_max_tokens, expected_value): """ - Test when max_output is not known => expect user_max_tokens - Test when max_output == max_input, @@ -819,9 +811,7 @@ def test_get_modified_max_tokens( if expected_value is None: assert calculated_value is None else: - assert ( - calculated_value == expected_value - ), "Got={}, Expected={}, Params={}".format( + assert calculated_value == expected_value, "Got={}, Expected={}, Params={}".format( calculated_value, expected_value, args ) @@ -840,12 +830,8 @@ def test_empty_tools(): reason="Skipping this test temporarily because it relies on a function being called that I am removing." ) def test_gpt_4o_token_counter(): - with patch.object( - litellm.utils, "openai_token_counter", new=MagicMock() - ) as mock_client: - token_counter( - model="gpt-4o-2024-05-13", messages=[{"role": "user", "content": "Hey!"}] - ) + with patch.object(litellm.utils, "openai_token_counter", new=MagicMock()) as mock_client: + token_counter(model="gpt-4o-2024-05-13", messages=[{"role": "user", "content": "Hey!"}]) mock_client.assert_called() @@ -974,9 +960,7 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("command-r-v1") # Verify the attempt to load Cohere tokenizer - mock_from_pretrained.assert_called_once_with( - "Xenova/c4ai-command-r-v01-tokenizer" - ) + mock_from_pretrained.assert_called_once_with("Xenova/c4ai-command-r-v01-tokenizer") # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") @@ -1009,9 +993,7 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("llama-2-7b") # Verify the attempt to load Llama-2 tokenizer - mock_from_pretrained.assert_called_once_with( - "hf-internal-testing/llama-tokenizer" - ) + mock_from_pretrained.assert_called_once_with("hf-internal-testing/llama-tokenizer") # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") @@ -1138,9 +1120,7 @@ def test_token_counter_with_anthropic_tool_use(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count: user message + "I'll check" text + "get_weather" name + input dict - assert ( - tokens > 15 - ), f"Expected reasonable token count for message with tool_use, got {tokens}" + assert tokens > 15, f"Expected reasonable token count for message with tool_use, got {tokens}" def test_token_counter_with_anthropic_tool_result(): @@ -1179,9 +1159,7 @@ def test_token_counter_with_anthropic_tool_result(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" - assert ( - tokens > 25 - ), f"Expected reasonable token count for conversation with tool_result, got {tokens}" + assert tokens > 25, f"Expected reasonable token count for conversation with tool_result, got {tokens}" def test_token_counter_with_nested_tool_result(): @@ -1215,9 +1193,7 @@ def test_token_counter_with_nested_tool_result(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count both nested text blocks - assert ( - tokens > 15 - ), f"Expected reasonable token count for nested tool_result, got {tokens}" + assert tokens > 15, f"Expected reasonable token count for nested tool_result, got {tokens}" def test_token_counter_tool_use_and_result_combined(): @@ -1280,9 +1256,7 @@ def test_token_counter_tool_use_and_result_combined(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count all text, tool names, inputs, and results - assert ( - tokens > 60 - ), f"Expected substantial token count for full tool conversation, got {tokens}" + assert tokens > 60, f"Expected substantial token count for full tool conversation, got {tokens}" def test_token_counter_with_image_url(): @@ -1333,12 +1307,8 @@ def test_token_counter_with_image_url(): } ] - tokens_str = token_counter( - model="gpt-3.5-turbo", messages=messages_str, use_default_image_token_count=True - ) - assert ( - tokens_str > 0 - ), f"Expected positive token count for string image_url, got {tokens_str}" + tokens_str = token_counter(model="gpt-3.5-turbo", messages=messages_str, use_default_image_token_count=True) + assert tokens_str > 0, f"Expected positive token count for string image_url, got {tokens_str}" # Test invalid detail value raises error messages_invalid = [ @@ -1359,9 +1329,7 @@ def test_token_counter_with_image_url(): with pytest.raises(ValueError, match="Invalid detail value") as exc_info: token_counter(model="gpt-3.5-turbo", messages=messages_invalid) e = exc_info.value - assert "Invalid detail value" in str( - e - ), f"Expected detail validation error, got: {e}" + assert "Invalid detail value" in str(e), f"Expected detail validation error, got: {e}" def test_token_counter_with_thinking_content(): @@ -1401,15 +1369,11 @@ def test_token_counter_with_thinking_content(): {"role": "user", "content": [{"type": "text", "text": "Thanks"}]}, ] - tokens = token_counter( - model="anthropic/claude-sonnet-4-5-20250929", messages=messages - ) + tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count: user message + thinking text + response text + "Thanks" # The thinking text alone is ~30 tokens, plus other content should be > 50 total - assert ( - tokens > 50 - ), f"Expected substantial token count for message with thinking, got {tokens}" + assert tokens > 50, f"Expected substantial token count for message with thinking, got {tokens}" # Test that thinking block without 'thinking' field doesn't crash (edge case) messages_no_thinking = [ @@ -1426,16 +1390,10 @@ def test_token_counter_with_thinking_content(): } ] - tokens_no_thinking = token_counter( - model="anthropic/claude-sonnet-4-5-20250929", messages=messages_no_thinking - ) - assert ( - tokens_no_thinking > 0 - ), f"Expected positive token count even with empty thinking, got {tokens_no_thinking}" + tokens_no_thinking = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=messages_no_thinking) + assert tokens_no_thinking > 0, f"Expected positive token count even with empty thinking, got {tokens_no_thinking}" # Should only count "Response" and message overhead - assert ( - tokens_no_thinking < 15 - ), f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}" + assert tokens_no_thinking < 15, f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}" def test_token_counter_with_redacted_thinking_content(): @@ -1456,6 +1414,7 @@ def test_token_counter_with_redacted_thinking_content(): assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search @@ -1478,9 +1437,7 @@ def test_token_counter_with_tool_reference_block(): ] # Must not raise, and must produce a positive token count. - tokens = token_counter_new( - model="anthropic/claude-sonnet-4-5-20250929", messages=messages - ) + tokens = token_counter_new(model="anthropic/claude-sonnet-4-5-20250929", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # A tool_reference with no/empty tool_name must also be handled gracefully. @@ -1490,9 +1447,7 @@ def test_token_counter_with_tool_reference_block(): "content": [{"type": "tool_reference", "tool_name": ""}], } ] - tokens_empty = token_counter_new( - model="anthropic/claude-sonnet-4-5-20250929", messages=messages_empty - ) + tokens_empty = token_counter_new(model="anthropic/claude-sonnet-4-5-20250929", messages=messages_empty) assert tokens_empty >= 0 @@ -1504,7 +1459,7 @@ def test_count_content_list_rejects_unknown_type(): """ from litellm.litellm_core_utils.token_counter import _count_content_list - with pytest.raises(ValueError, match='Error getting number of tokens from content list: Invalid') as exc_info: + with pytest.raises(ValueError, match="Error getting number of tokens from content list: Invalid") as exc_info: _count_content_list( count_function=len, content_list=[{"type": "totally_unknown_block"}], @@ -1545,9 +1500,7 @@ def test_token_counter_with_anthropic_image_block(source: dict[str, str]): messages=messages, use_default_image_token_count=True, ) - assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, ( - f"Expected the image block to contribute tokens, got {tokens}" - ) + assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, f"Expected the image block to contribute tokens, got {tokens}" def test_anthropic_image_block_matches_equivalent_image_url(): @@ -1579,12 +1532,8 @@ def test_anthropic_image_block_matches_equivalent_image_url(): } ] - anthropic_tokens = token_counter( - model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages - ) - openai_tokens = token_counter( - model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages - ) + anthropic_tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages) + openai_tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages) assert anthropic_tokens == openai_tokens @@ -1642,9 +1591,7 @@ def test_anthropic_image_block_with_empty_base64_data(): tokens = _count_content_list( count_function=len, - content_list=[ - {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} - ], + content_list=[{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}], use_default_image_token_count=False, default_token_count=None, ) @@ -1756,7 +1703,10 @@ def test_openai_file_block_prices_like_the_equivalent_anthropic_document(): assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt]) readable: Final = _pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",)) - readable_file: Final = {"type": "file", "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + readable}} + readable_file: Final = { + "type": "file", + "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + readable}, + } readable_document: Final = {"type": "document", "title": "report.pdf", "source": _pdf_source(readable)} assert _count_user_content([prompt, readable_file]) == _count_user_content([prompt, readable_document]) assert _count_user_content([prompt, readable_file]) > _count_user_content([prompt, inline_file]) @@ -1770,9 +1720,7 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): assert _count_user_content([prompt, by_id]) == _count_user_content([prompt]) named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}} - assert _count_user_content([prompt, named]) == _count_user_content( - [prompt, {"type": "text", "text": "report.pdf"}] - ) + assert _count_user_content([prompt, named]) == _count_user_content([prompt, {"type": "text", "text": "report.pdf"}]) def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: @@ -1811,12 +1759,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> def _pdf_stream(content: bytes, dictionary: bytes = b"") -> bytes: dictionary_entries: Final = b" " + dictionary if dictionary else b"" return ( - b"<< /Length " - + str(len(content)).encode() - + dictionary_entries - + b" >>\nstream\n" - + content - + b"\nendstream" + b"<< /Length " + str(len(content)).encode() + dictionary_entries + b" >>\nstream\n" + content + b"\nendstream" ) @@ -1831,15 +1774,12 @@ def _pdf_data_url_with_xobjects( b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>", font_object, b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " - b"/Resources << /Font << /F1 3 0 R >> /XObject " - + page_xobjects - + b" >> /Contents 5 0 R >>", + b"/Resources << /Font << /F1 3 0 R >> /XObject " + page_xobjects + b" >> /Contents 5 0 R >>", _pdf_stream(page_content), *xobject_objects, ) object_chunks: Final = tuple( - f"{index} 0 obj\n".encode() + body + b"\nendobj\n" - for index, body in enumerate(object_bodies, start=1) + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" for index, body in enumerate(object_bodies, start=1) ) header: Final = b"%PDF-1.4\n" offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) @@ -1858,11 +1798,7 @@ def _pdf_data_url_with_doubling_forms(depth: int, leaf_content: bytes) -> str: _pdf_stream( leaf_content if index == depth - 1 else b"/Fm Do\n/Fm Do\n", b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject " - + ( - b"<< >>" - if index == depth - 1 - else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>" - ) + + (b"<< >>" if index == depth - 1 else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>") + b" >>", ) for index in range(depth) @@ -1930,8 +1866,7 @@ def _pdf_data_url_with_shared_xobjects( first_xobject_object_number: Final = 5 + 2 * page_count page_references: Final = b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) xobject_references: Final = b" ".join( - f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() - for index in range(len(xobject_objects)) + f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() for index in range(len(xobject_objects)) ) page_objects: Final = tuple( b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources 4 0 R /Contents " @@ -1941,9 +1876,7 @@ def _pdf_data_url_with_shared_xobjects( content_objects: Final = tuple(_pdf_stream(content) for content in page_contents) object_bodies: Final = ( b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [" - + page_references - + f"] /Count {page_count} >>".encode(), + b"<< /Type /Pages /Kids [" + page_references + f"] /Count {page_count} >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", b"<< /Font << /F1 3 0 R >> /XObject << " + xobject_references + b" >> >>", *page_objects, @@ -1951,8 +1884,7 @@ def _pdf_data_url_with_shared_xobjects( *xobject_objects, ) object_chunks: Final = tuple( - f"{index} 0 obj\n".encode() + body + b"\nendobj\n" - for index, body in enumerate(object_bodies, start=1) + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" for index, body in enumerate(object_bodies, start=1) ) header: Final = b"%PDF-1.4\n" offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) @@ -2047,13 +1979,18 @@ def test_base64_pdf_document_counts_every_page_text_and_rendering(): first: Final = "Revenue grew eleven percent while churn fell to two percent." second: Final = "Headcount is flat and the office lease was renewed for three years." base: Final = _count_user_content([prompt]) - blank_page: Final = _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64(("",)))}]) - base + blank_page: Final = ( + _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64(("",)))}]) - base + ) assert blank_page > 0 assert _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first, second)))}]) == ( - _count_user_content([prompt, {"type": "text", "text": first}, {"type": "text", "text": second}]) + 2 * blank_page + _count_user_content([prompt, {"type": "text", "text": first}, {"type": "text", "text": second}]) + + 2 * blank_page + ) + one_page: Final = ( + _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,)))}]) - base ) - one_page: Final = _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,)))}]) - base twelve_pages: Final = ( _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,) * 12))}]) - base ) @@ -2061,7 +1998,9 @@ def test_base64_pdf_document_counts_every_page_text_and_rendering(): assert twelve_pages == 12 * one_page -@pytest.mark.parametrize("fields", [{}, {"title": "Q3 board packet", "context": "Shared by finance"}], ids=["bare", "described"]) +@pytest.mark.parametrize( + "fields", [{}, {"title": "Q3 board packet", "context": "Shared by finance"}], ids=["bare", "described"] +) def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[str, str]): """Anthropic rasterizes each PDF page within its image limits (1568 px long edge, 1.15 MP) and bills width * height / 750 tokens for it: https://platform.claude.com/docs/en/build-with-claude/pdf-support and @@ -2306,8 +2245,7 @@ def test_pdf_image_xobject_does_not_trigger_the_content_limit(): image_data: Final = b"\x00" * (1024 * 1024) image: Final = _pdf_stream( image_data, - b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 " - b"/ColorSpace /DeviceRGB /BitsPerComponent 8", + b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 /ColorSpace /DeviceRGB /BitsPerComponent 8", ) data_url: Final = _pdf_data_url_with_xobjects( page_text, @@ -2591,6 +2529,76 @@ def test_oversized_pdf_text_uses_a_sampled_token_estimate(): assert expected * 4 // 5 <= priced <= expected * 6 // 5 +def test_many_tiny_pdf_text_objects_price_within_sample_estimate(): + repetitions: Final = 200_000 + text_object: Final = b"BT /F1 12 Tf (x) Tj ET\n" + page: Final = _fake_pdf_page(text_object * repetitions) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = repetitions + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_oversized_literal_cut_after_escape_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf " + suffix: Final = b" Tj ET" + operand_prefix_length: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(text_prefix) - len(suffix) + operand: Final = b"(" + b"x" * (operand_prefix_length - 2) + b"\\(" + b"x" * 128 + b")" + page: Final = _fake_pdf_page(text_prefix + operand + suffix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = operand.count(b"x") + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_oversized_text_cut_inside_trailing_comment_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf (x) Tj " + comment: Final = b"%" + b"comment" * PDF_PAGE_MAX_EXACT_CONTENT_BYTES + page: Final = _fake_pdf_page(text_prefix + comment + b"\nET") + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert priced - blank_price == 1 + + +def test_oversized_tj_array_cut_inside_comment_stays_extractable(): + repetitions: Final = 1_000 + content: Final = ( + b"BT /F1 12 Tf [" + + b"(x) 0 " * repetitions + + b"% comment with [" + + b"comment" * PDF_PAGE_MAX_EXACT_CONTENT_BYTES + + b"\n] TJ ET" + ) + page: Final = _fake_pdf_page(content) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert repetitions * 3 // 4 <= priced - blank_price <= repetitions * 5 // 4 + + def test_oversized_identity_h_pdf_uses_tounicode_for_two_byte_glyphs(): repetitions: Final = 1_000 path_operator: Final = b"10 10 m 20 20 l S\n" @@ -2785,11 +2793,15 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. def _png_data_url(width: int, height: int) -> str: - ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + ihdr = ( + b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") + ) return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() -@pytest.mark.parametrize(("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)]) +@pytest.mark.parametrize( + ("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)] +) def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, height: int) -> None: assert calculate_img_tokens(_png_data_url(width, height), mode="high") <= high_detail_image_token_upper_bound() @@ -2820,7 +2832,9 @@ def handle(request): huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) """ -HUB_TOKENIZER_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +HUB_TOKENIZER_SCRIPT: Final = ( + HUB_SETUP_SCRIPT + + """ litellm.cohere_models = {"command-r-v1"} litellm.anthropic_models = {"claude-2"} custom = litellm.create_pretrained_tokenizer("Xenova/llama-3-tokenizer") @@ -2833,11 +2847,15 @@ print(json.dumps({ "requested": sorted(set(requested)), })) """ +) -HUB_ROUND_TRIP_SCRIPT: Final = HUB_SETUP_SCRIPT + """ +HUB_ROUND_TRIP_SCRIPT: Final = ( + HUB_SETUP_SCRIPT + + """ tokens = litellm.encode(model="meta-llama/Llama-2-7b-chat", text=text) print(json.dumps({"tokens": tokens, "decoded": litellm.decode(model="meta-llama/Llama-2-7b-chat", tokens=tokens), "requested": sorted(set(requested))})) """ +) def _word_level_tokenizer_json( @@ -2912,7 +2930,9 @@ def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: def _threshold_test_messages(turns: int) -> list[dict]: messages: list[dict] = [{"role": "system", "content": "You are a terse assistant. " * 20}] for index in range(turns): - messages.append({"role": "user", "content": f"Question {index}: what is the capital of country number {index}?"}) + messages.append( + {"role": "user", "content": f"Question {index}: what is the capital of country number {index}?"} + ) messages.append( { "role": "assistant", @@ -2999,7 +3019,9 @@ _ANTHROPIC_SHAPE_MESSAGES: Final = [ }, { "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "file body line\n" * 40}]}], + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "file body line\n" * 40}]} + ], }, {"role": "assistant", "content": [{"type": "text", "text": "Here is what the file does. " * 20}]}, ] From 5b78b96394202bbf5906ec8e7c823beea3f7ccb3 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 00:55:12 +0000 Subject: [PATCH 16/21] test(token_counter): drop unrelated formatting churn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/test_token_counter.py | 200 +++++++++++------- 1 file changed, 124 insertions(+), 76 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 61e155cae7e..0119cad6164 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -386,7 +386,9 @@ def test_token_counter_legacy_function_call_counts_arguments(): MESSAGES_TEXT, ) def test_token_counter_textonly(message_count_pair): - counted_tokens = token_counter(model="gpt-35-turbo", messages=[message_count_pair["message"]]) + counted_tokens = token_counter( + model="gpt-35-turbo", messages=[message_count_pair["message"]] + ) assert counted_tokens == message_count_pair["count"] @@ -410,7 +412,9 @@ def test_token_counter_count_response_tokens(message_count_pair): MESSAGES_WITH_IMAGES, ) def test_token_counter_with_images(message_count_pair): - counted_tokens = token_counter(model="gpt-4o", messages=[message_count_pair["message"]]) + counted_tokens = token_counter( + model="gpt-4o", messages=[message_count_pair["message"]] + ) assert counted_tokens == message_count_pair["count"] @@ -433,16 +437,18 @@ def test_token_counter_with_tools(message_count_pair): pass # expected else: tolerated_diff = message_count_pair["count-tolerate"] - expected_tokens - assert actual_diff <= tolerated_diff, ( - f"Expected {expected_tokens} tokens, got {counted_tokens}. Counted tokens is only allowed to be off by {tolerated_diff} in the over-counting direction." - ) + assert ( + actual_diff <= tolerated_diff + ), f"Expected {expected_tokens} tokens, got {counted_tokens}. Counted tokens is only allowed to be off by {tolerated_diff} in the over-counting direction." if actual_diff != tolerated_diff: raise NeedsToleranceUpdateError( f"SOMETHING BROKEN GOT FIXED! THIS is good! Adjust 'count-tolerate' from {message_count_pair['count-tolerate']} to {counted_tokens}" ) else: - assert expected_tokens == counted_tokens, f"Expected {expected_tokens} tokens, got {counted_tokens}." + assert ( + expected_tokens == counted_tokens + ), f"Expected {expected_tokens} tokens, got {counted_tokens}." def test_token_counter_counts_gemini_function_declarations(): @@ -778,7 +784,9 @@ def test_load_test_token_counter(model): ("gpt-3.5-turbo", "gpt-3.5-turbo", 4000, 5000, 4096), # model max output = 4096 ], ) -def test_get_modified_max_tokens(model, base_model, input_tokens, user_max_tokens, expected_value): +def test_get_modified_max_tokens( + model, base_model, input_tokens, user_max_tokens, expected_value +): """ - Test when max_output is not known => expect user_max_tokens - Test when max_output == max_input, @@ -811,7 +819,9 @@ def test_get_modified_max_tokens(model, base_model, input_tokens, user_max_token if expected_value is None: assert calculated_value is None else: - assert calculated_value == expected_value, "Got={}, Expected={}, Params={}".format( + assert ( + calculated_value == expected_value + ), "Got={}, Expected={}, Params={}".format( calculated_value, expected_value, args ) @@ -830,8 +840,12 @@ def test_empty_tools(): reason="Skipping this test temporarily because it relies on a function being called that I am removing." ) def test_gpt_4o_token_counter(): - with patch.object(litellm.utils, "openai_token_counter", new=MagicMock()) as mock_client: - token_counter(model="gpt-4o-2024-05-13", messages=[{"role": "user", "content": "Hey!"}]) + with patch.object( + litellm.utils, "openai_token_counter", new=MagicMock() + ) as mock_client: + token_counter( + model="gpt-4o-2024-05-13", messages=[{"role": "user", "content": "Hey!"}] + ) mock_client.assert_called() @@ -960,7 +974,9 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("command-r-v1") # Verify the attempt to load Cohere tokenizer - mock_from_pretrained.assert_called_once_with("Xenova/c4ai-command-r-v01-tokenizer") + mock_from_pretrained.assert_called_once_with( + "Xenova/c4ai-command-r-v01-tokenizer" + ) # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") @@ -993,7 +1009,9 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("llama-2-7b") # Verify the attempt to load Llama-2 tokenizer - mock_from_pretrained.assert_called_once_with("hf-internal-testing/llama-tokenizer") + mock_from_pretrained.assert_called_once_with( + "hf-internal-testing/llama-tokenizer" + ) # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") @@ -1120,7 +1138,9 @@ def test_token_counter_with_anthropic_tool_use(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count: user message + "I'll check" text + "get_weather" name + input dict - assert tokens > 15, f"Expected reasonable token count for message with tool_use, got {tokens}" + assert ( + tokens > 15 + ), f"Expected reasonable token count for message with tool_use, got {tokens}" def test_token_counter_with_anthropic_tool_result(): @@ -1159,7 +1179,9 @@ def test_token_counter_with_anthropic_tool_result(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" - assert tokens > 25, f"Expected reasonable token count for conversation with tool_result, got {tokens}" + assert ( + tokens > 25 + ), f"Expected reasonable token count for conversation with tool_result, got {tokens}" def test_token_counter_with_nested_tool_result(): @@ -1193,7 +1215,9 @@ def test_token_counter_with_nested_tool_result(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count both nested text blocks - assert tokens > 15, f"Expected reasonable token count for nested tool_result, got {tokens}" + assert ( + tokens > 15 + ), f"Expected reasonable token count for nested tool_result, got {tokens}" def test_token_counter_tool_use_and_result_combined(): @@ -1256,7 +1280,9 @@ def test_token_counter_tool_use_and_result_combined(): tokens = token_counter(model="gpt-3.5-turbo", messages=messages) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count all text, tool names, inputs, and results - assert tokens > 60, f"Expected substantial token count for full tool conversation, got {tokens}" + assert ( + tokens > 60 + ), f"Expected substantial token count for full tool conversation, got {tokens}" def test_token_counter_with_image_url(): @@ -1307,8 +1333,12 @@ def test_token_counter_with_image_url(): } ] - tokens_str = token_counter(model="gpt-3.5-turbo", messages=messages_str, use_default_image_token_count=True) - assert tokens_str > 0, f"Expected positive token count for string image_url, got {tokens_str}" + tokens_str = token_counter( + model="gpt-3.5-turbo", messages=messages_str, use_default_image_token_count=True + ) + assert ( + tokens_str > 0 + ), f"Expected positive token count for string image_url, got {tokens_str}" # Test invalid detail value raises error messages_invalid = [ @@ -1329,7 +1359,9 @@ def test_token_counter_with_image_url(): with pytest.raises(ValueError, match="Invalid detail value") as exc_info: token_counter(model="gpt-3.5-turbo", messages=messages_invalid) e = exc_info.value - assert "Invalid detail value" in str(e), f"Expected detail validation error, got: {e}" + assert "Invalid detail value" in str( + e + ), f"Expected detail validation error, got: {e}" def test_token_counter_with_thinking_content(): @@ -1369,11 +1401,15 @@ def test_token_counter_with_thinking_content(): {"role": "user", "content": [{"type": "text", "text": "Thanks"}]}, ] - tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=messages) + tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=messages + ) assert tokens > 0, f"Expected positive token count, got {tokens}" # Should count: user message + thinking text + response text + "Thanks" # The thinking text alone is ~30 tokens, plus other content should be > 50 total - assert tokens > 50, f"Expected substantial token count for message with thinking, got {tokens}" + assert ( + tokens > 50 + ), f"Expected substantial token count for message with thinking, got {tokens}" # Test that thinking block without 'thinking' field doesn't crash (edge case) messages_no_thinking = [ @@ -1390,10 +1426,16 @@ def test_token_counter_with_thinking_content(): } ] - tokens_no_thinking = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=messages_no_thinking) - assert tokens_no_thinking > 0, f"Expected positive token count even with empty thinking, got {tokens_no_thinking}" + tokens_no_thinking = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=messages_no_thinking + ) + assert ( + tokens_no_thinking > 0 + ), f"Expected positive token count even with empty thinking, got {tokens_no_thinking}" # Should only count "Response" and message overhead - assert tokens_no_thinking < 15, f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}" + assert ( + tokens_no_thinking < 15 + ), f"Expected minimal token count for empty thinking block, got {tokens_no_thinking}" def test_token_counter_with_redacted_thinking_content(): @@ -1414,7 +1456,6 @@ def test_token_counter_with_redacted_thinking_content(): assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) - def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search @@ -1437,7 +1478,9 @@ def test_token_counter_with_tool_reference_block(): ] # Must not raise, and must produce a positive token count. - tokens = token_counter_new(model="anthropic/claude-sonnet-4-5-20250929", messages=messages) + tokens = token_counter_new( + model="anthropic/claude-sonnet-4-5-20250929", messages=messages + ) assert tokens > 0, f"Expected positive token count, got {tokens}" # A tool_reference with no/empty tool_name must also be handled gracefully. @@ -1447,7 +1490,9 @@ def test_token_counter_with_tool_reference_block(): "content": [{"type": "tool_reference", "tool_name": ""}], } ] - tokens_empty = token_counter_new(model="anthropic/claude-sonnet-4-5-20250929", messages=messages_empty) + tokens_empty = token_counter_new( + model="anthropic/claude-sonnet-4-5-20250929", messages=messages_empty + ) assert tokens_empty >= 0 @@ -1459,7 +1504,7 @@ def test_count_content_list_rejects_unknown_type(): """ from litellm.litellm_core_utils.token_counter import _count_content_list - with pytest.raises(ValueError, match="Error getting number of tokens from content list: Invalid") as exc_info: + with pytest.raises(ValueError, match='Error getting number of tokens from content list: Invalid') as exc_info: _count_content_list( count_function=len, content_list=[{"type": "totally_unknown_block"}], @@ -1500,7 +1545,9 @@ def test_token_counter_with_anthropic_image_block(source: dict[str, str]): messages=messages, use_default_image_token_count=True, ) - assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, f"Expected the image block to contribute tokens, got {tokens}" + assert tokens > DEFAULT_IMAGE_TOKEN_COUNT, ( + f"Expected the image block to contribute tokens, got {tokens}" + ) def test_anthropic_image_block_matches_equivalent_image_url(): @@ -1532,8 +1579,12 @@ def test_anthropic_image_block_matches_equivalent_image_url(): } ] - anthropic_tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages) - openai_tokens = token_counter(model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages) + anthropic_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=anthropic_messages + ) + openai_tokens = token_counter( + model="anthropic/claude-sonnet-4-5-20250929", messages=openai_messages + ) assert anthropic_tokens == openai_tokens @@ -1591,7 +1642,9 @@ def test_anthropic_image_block_with_empty_base64_data(): tokens = _count_content_list( count_function=len, - content_list=[{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}], + content_list=[ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}} + ], use_default_image_token_count=False, default_token_count=None, ) @@ -1703,10 +1756,7 @@ def test_openai_file_block_prices_like_the_equivalent_anthropic_document(): assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt]) readable: Final = _pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",)) - readable_file: Final = { - "type": "file", - "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + readable}, - } + readable_file: Final = {"type": "file", "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64," + readable}} readable_document: Final = {"type": "document", "title": "report.pdf", "source": _pdf_source(readable)} assert _count_user_content([prompt, readable_file]) == _count_user_content([prompt, readable_document]) assert _count_user_content([prompt, readable_file]) > _count_user_content([prompt, inline_file]) @@ -1720,7 +1770,9 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): assert _count_user_content([prompt, by_id]) == _count_user_content([prompt]) named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}} - assert _count_user_content([prompt, named]) == _count_user_content([prompt, {"type": "text", "text": "report.pdf"}]) + assert _count_user_content([prompt, named]) == _count_user_content( + [prompt, {"type": "text", "text": "report.pdf"}] + ) def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: @@ -1759,7 +1811,12 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> def _pdf_stream(content: bytes, dictionary: bytes = b"") -> bytes: dictionary_entries: Final = b" " + dictionary if dictionary else b"" return ( - b"<< /Length " + str(len(content)).encode() + dictionary_entries + b" >>\nstream\n" + content + b"\nendstream" + b"<< /Length " + + str(len(content)).encode() + + dictionary_entries + + b" >>\nstream\n" + + content + + b"\nendstream" ) @@ -1774,12 +1831,15 @@ def _pdf_data_url_with_xobjects( b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>", font_object, b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " - b"/Resources << /Font << /F1 3 0 R >> /XObject " + page_xobjects + b" >> /Contents 5 0 R >>", + b"/Resources << /Font << /F1 3 0 R >> /XObject " + + page_xobjects + + b" >> /Contents 5 0 R >>", _pdf_stream(page_content), *xobject_objects, ) object_chunks: Final = tuple( - f"{index} 0 obj\n".encode() + body + b"\nendobj\n" for index, body in enumerate(object_bodies, start=1) + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) ) header: Final = b"%PDF-1.4\n" offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) @@ -1798,7 +1858,11 @@ def _pdf_data_url_with_doubling_forms(depth: int, leaf_content: bytes) -> str: _pdf_stream( leaf_content if index == depth - 1 else b"/Fm Do\n/Fm Do\n", b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject " - + (b"<< >>" if index == depth - 1 else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>") + + ( + b"<< >>" + if index == depth - 1 + else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>" + ) + b" >>", ) for index in range(depth) @@ -1866,7 +1930,8 @@ def _pdf_data_url_with_shared_xobjects( first_xobject_object_number: Final = 5 + 2 * page_count page_references: Final = b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) xobject_references: Final = b" ".join( - f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() for index in range(len(xobject_objects)) + f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() + for index in range(len(xobject_objects)) ) page_objects: Final = tuple( b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources 4 0 R /Contents " @@ -1876,7 +1941,9 @@ def _pdf_data_url_with_shared_xobjects( content_objects: Final = tuple(_pdf_stream(content) for content in page_contents) object_bodies: Final = ( b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [" + page_references + f"] /Count {page_count} >>".encode(), + b"<< /Type /Pages /Kids [" + + page_references + + f"] /Count {page_count} >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", b"<< /Font << /F1 3 0 R >> /XObject << " + xobject_references + b" >> >>", *page_objects, @@ -1884,7 +1951,8 @@ def _pdf_data_url_with_shared_xobjects( *xobject_objects, ) object_chunks: Final = tuple( - f"{index} 0 obj\n".encode() + body + b"\nendobj\n" for index, body in enumerate(object_bodies, start=1) + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) ) header: Final = b"%PDF-1.4\n" offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) @@ -1979,18 +2047,13 @@ def test_base64_pdf_document_counts_every_page_text_and_rendering(): first: Final = "Revenue grew eleven percent while churn fell to two percent." second: Final = "Headcount is flat and the office lease was renewed for three years." base: Final = _count_user_content([prompt]) - blank_page: Final = ( - _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64(("",)))}]) - base - ) + blank_page: Final = _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64(("",)))}]) - base assert blank_page > 0 assert _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first, second)))}]) == ( - _count_user_content([prompt, {"type": "text", "text": first}, {"type": "text", "text": second}]) - + 2 * blank_page - ) - one_page: Final = ( - _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,)))}]) - base + _count_user_content([prompt, {"type": "text", "text": first}, {"type": "text", "text": second}]) + 2 * blank_page ) + one_page: Final = _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,)))}]) - base twelve_pages: Final = ( _count_user_content([prompt, {"type": "document", "source": _pdf_source(_pdf_base64((first,) * 12))}]) - base ) @@ -1998,9 +2061,7 @@ def test_base64_pdf_document_counts_every_page_text_and_rendering(): assert twelve_pages == 12 * one_page -@pytest.mark.parametrize( - "fields", [{}, {"title": "Q3 board packet", "context": "Shared by finance"}], ids=["bare", "described"] -) +@pytest.mark.parametrize("fields", [{}, {"title": "Q3 board packet", "context": "Shared by finance"}], ids=["bare", "described"]) def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[str, str]): """Anthropic rasterizes each PDF page within its image limits (1568 px long edge, 1.15 MP) and bills width * height / 750 tokens for it: https://platform.claude.com/docs/en/build-with-claude/pdf-support and @@ -2245,7 +2306,8 @@ def test_pdf_image_xobject_does_not_trigger_the_content_limit(): image_data: Final = b"\x00" * (1024 * 1024) image: Final = _pdf_stream( image_data, - b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 /ColorSpace /DeviceRGB /BitsPerComponent 8", + b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 " + b"/ColorSpace /DeviceRGB /BitsPerComponent 8", ) data_url: Final = _pdf_data_url_with_xobjects( page_text, @@ -2793,15 +2855,11 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. def _png_data_url(width: int, height: int) -> str: - ihdr = ( - b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") - ) + ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() -@pytest.mark.parametrize( - ("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)] -) +@pytest.mark.parametrize(("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)]) def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, height: int) -> None: assert calculate_img_tokens(_png_data_url(width, height), mode="high") <= high_detail_image_token_upper_bound() @@ -2832,9 +2890,7 @@ def handle(request): huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) """ -HUB_TOKENIZER_SCRIPT: Final = ( - HUB_SETUP_SCRIPT - + """ +HUB_TOKENIZER_SCRIPT: Final = HUB_SETUP_SCRIPT + """ litellm.cohere_models = {"command-r-v1"} litellm.anthropic_models = {"claude-2"} custom = litellm.create_pretrained_tokenizer("Xenova/llama-3-tokenizer") @@ -2847,15 +2903,11 @@ print(json.dumps({ "requested": sorted(set(requested)), })) """ -) -HUB_ROUND_TRIP_SCRIPT: Final = ( - HUB_SETUP_SCRIPT - + """ +HUB_ROUND_TRIP_SCRIPT: Final = HUB_SETUP_SCRIPT + """ tokens = litellm.encode(model="meta-llama/Llama-2-7b-chat", text=text) print(json.dumps({"tokens": tokens, "decoded": litellm.decode(model="meta-llama/Llama-2-7b-chat", tokens=tokens), "requested": sorted(set(requested))})) """ -) def _word_level_tokenizer_json( @@ -2930,9 +2982,7 @@ def test_empty_custom_tokenizer_uses_model_tokenizer() -> None: def _threshold_test_messages(turns: int) -> list[dict]: messages: list[dict] = [{"role": "system", "content": "You are a terse assistant. " * 20}] for index in range(turns): - messages.append( - {"role": "user", "content": f"Question {index}: what is the capital of country number {index}?"} - ) + messages.append({"role": "user", "content": f"Question {index}: what is the capital of country number {index}?"}) messages.append( { "role": "assistant", @@ -3019,9 +3069,7 @@ _ANTHROPIC_SHAPE_MESSAGES: Final = [ }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "file body line\n" * 40}]} - ], + "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "file body line\n" * 40}]}], }, {"role": "assistant", "content": [{"type": "text", "text": "Here is what the file does. " * 20}]}, ] From 9ed0bcac8ed27f6880f60b4481967d19b32e0e15 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 01:37:59 +0000 Subject: [PATCH 17/21] fix(token_counter): satisfy PDF CI gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 6 ++++-- pyproject.toml | 1 + uv.lock | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 1e4c26b26ea..07455d8c901 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1118,7 +1118,7 @@ _PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool] _PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]] -def _pdf_text_object_records( +def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream PDF traversal metrics: _PdfStreamMetrics, invocation_path: tuple[int, ...] = (), ) -> Iterable[_PdfTextObjectRecord]: @@ -1240,7 +1240,9 @@ def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes: return separator + text_object + closure -def _pdf_truncated_text_sample_closures(prefix: bytes, include_text_show: bool) -> tuple[int, bytes]: +def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax-state scanner + prefix: bytes, include_text_show: bool +) -> tuple[int, bytes]: comment_open: bool = False # rebind-ok: scans one bounded PDF prefix escape_open: bool = False # rebind-ok: scans one bounded PDF prefix literal_depth: int = 0 # rebind-ok: tracks balanced literal strings diff --git a/pyproject.toml b/pyproject.toml index 9076760153a..c3524bcacfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -304,6 +304,7 @@ healthcheck = [ benchmarks = [ "pytest==9.0.3", "pytest-codspeed==4.3.0", + "pypdf>=6.16.1,<7.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", ] diff --git a/uv.lock b/uv.lock index 03403828225..469088f167a 100644 --- a/uv.lock +++ b/uv.lock @@ -4757,6 +4757,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -4980,6 +4981,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.16.1,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ] From 9a2d6d99343beec1399b3049087f09dc03a0d30b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 02:20:10 +0000 Subject: [PATCH 18/21] fix(token_counter): track real ET closure for PDF text samples Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 107 +++++++++++------- .../litellm_core_utils/test_token_counter.py | 32 ++++++ 2 files changed, 99 insertions(+), 40 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 07455d8c901..25b51e394b2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -368,7 +368,7 @@ PDF_TEXT_OBJECT: Final = re.compile( + rb"(?:(?:" + PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|[\s\S])*?" - + rb"(?:(?<=\s)ET(?![^\s\[\]()<>{}/%])|\Z)))" + + rb"(?:(?P(?<=\s)ET(?![^\s\[\]()<>{}/%]))|\Z)))" ) PDF_XOBJECT_INVOCATION: Final = re.compile( PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" @@ -963,6 +963,7 @@ class _PdfTextObjectRecord: end: int first_in_stream: bool truncated: bool + closed: bool _PDF_EMPTY_RESOURCES: Final[_PdfResourceDictionary] = ( @@ -1113,7 +1114,7 @@ def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: ) -_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int, bool], ...] +_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int, bool, bool], ...] _PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool] _PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]] @@ -1125,19 +1126,20 @@ def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream P text_object_span_memo: Final[dict[int, _PdfTextObjectSpans]] = {} # mutable-ok: page-local regex cache subtree_text_memo: Final[dict[int, bool]] = {} # mutable-ok: page-local subtree cache - def partial_text_object_end(content: bytes, start: int) -> int: + def partial_text_object_end(content: bytes, start: int) -> tuple[int, bool, bool]: end: Final = min(len(content), start + PDF_PAGE_MAX_EXACT_CONTENT_BYTES) bounded_content: Final = content[start:end] - return next( - ( - start + match.end("text_object") - for match in PDF_TEXT_OBJECT.finditer(bounded_content) - if match.group("text_object") is not None - ), - start, + match: Final = next( + (match for match in PDF_TEXT_OBJECT.finditer(bounded_content) if match.group("text_object") is not None), + None, ) + if match is None: + return start, False, False + closed: Final = match.group("text_object_end") is not None + truncated: Final = match.end("text_object") == len(bounded_content) and end < len(content) and not closed + return start + match.end("text_object"), closed, truncated - def text_object_spans_in_window(content: bytes, window_end: int) -> Iterable[tuple[int, int, bool]]: + def text_object_spans_in_window(content: bytes, window_end: int) -> Iterable[tuple[int, int, bool, bool]]: window_content: Final = content[:window_end] for match in PDF_TEXT_OBJECT.finditer(window_content): if match.group("text_object") is None: @@ -1145,31 +1147,34 @@ def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream P if ( match.end("text_object") == window_end and window_end < len(content) - and content[match.end("text_object") - 2 : match.end("text_object")] != b"ET" + and match.group("text_object_end") is None ): + partial_end, partial_closed, partial_truncated = partial_text_object_end( + content, match.start("text_object") + ) yield ( match.start("text_object"), - partial_text_object_end(content, match.start("text_object")), - True, + partial_end, + partial_truncated, + partial_closed, ) return yield ( match.start("text_object"), match.end("text_object"), - content[match.end("text_object") - 2 : match.end("text_object")] != b"ET", + match.group("text_object_end") is None, + match.group("text_object_end") is not None, ) def scan_text_object_spans( content: bytes, window_end: int, ) -> tuple[_PdfTextObjectSpans, int]: - spans: Final[list[tuple[int, int, bool]]] = [] # mutable-ok: capped page-local span cache + spans: Final[list[tuple[int, int, bool, bool]]] = [] # mutable-ok: capped page-local span cache cumulative_length: int = 0 # rebind-ok: accumulates record lengths until the sample cap - for start, end, truncated in text_object_spans_in_window(content, window_end): - spans.append((start, end, truncated)) - cumulative_length += ( - end - start + (len(b"\nET") if truncated else 0) + (0 if len(spans) == 1 else len(b"\n")) - ) + for start, end, truncated, closed in text_object_spans_in_window(content, window_end): + spans.append((start, end, truncated, closed)) + cumulative_length += end - start + (0 if closed else len(b"\nET")) + (0 if len(spans) == 1 else len(b"\n")) if cumulative_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: break return tuple(spans), cumulative_length @@ -1216,8 +1221,8 @@ def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream P pending_records: Final[list[_PdfTextRecordWalkItem]] = [(metrics, invocation_path)] # mutable-ok: DFS stack while pending_records: current_metrics, current_path = pending_records.pop() - for index, (start, end, truncated) in enumerate(text_object_spans_for(current_metrics)): - yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0, truncated) + for index, (start, end, truncated, closed) in enumerate(text_object_spans_for(current_metrics)): + yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0, truncated, closed) pending_records.extend( (form_metrics, current_path + (index,)) for index, form_metrics in reversed(tuple(enumerate(current_metrics.forms))) @@ -1227,21 +1232,33 @@ def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream P def _pdf_text_object_record_length(record: _PdfTextObjectRecord) -> int: text_object_length: Final = record.end - record.start - is_closed: Final = record.stream.content[record.end - 2 : record.end] == b"ET" - closure_bytes: Final = 0 if is_closed else len(b"\nET") separator_bytes: Final = 0 if record.first_in_stream else len(b"\n") - return text_object_length + closure_bytes + separator_bytes + if record.closed: + return text_object_length + separator_bytes + text_object: Final = record.stream.content[record.start : record.end] + prefix_length, closures = _pdf_truncated_text_sample_closures( + text_object, + include_text_show=True, + maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - separator_bytes - len(b"\nET"), + ) + return prefix_length + len(closures) + len(b"\nET") + separator_bytes def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes: text_object: Final = record.stream.content[record.start : record.end] separator: Final = b"" if record.first_in_stream else b"\n" - closure: Final = b"" if text_object.endswith(b"ET") else b"\nET" - return separator + text_object + closure + if record.closed: + return separator + text_object + prefix_length, closures = _pdf_truncated_text_sample_closures( + text_object, + include_text_show=True, + maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(separator) - len(b"\nET"), + ) + return separator + text_object[:prefix_length] + closures + b"\nET" def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax-state scanner - prefix: bytes, include_text_show: bool + prefix: bytes, include_text_show: bool, maximum_length: int | None = None ) -> tuple[int, bytes]: comment_open: bool = False # rebind-ok: scans one bounded PDF prefix escape_open: bool = False # rebind-ok: scans one bounded PDF prefix @@ -1250,6 +1267,7 @@ def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax array_depth: int = 0 # rebind-ok: tracks arrays outside strings bare_string_complete: bool = False # rebind-ok: tracks the final string operand best_prefix_length: int = 0 # rebind-ok: retains the latest prefix whose closures fit the sample cap + sample_limit: Final = len(prefix) if maximum_length is None else maximum_length best_state: tuple[bool, bool, int, bool, int, bool] = ( # rebind-ok: scanner state False, False, @@ -1293,15 +1311,22 @@ def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax bare_string_complete = False elif byte not in b" \t\r\n\f\x00": bare_string_complete = False - if index + 1 + (1 if comment_open else 0) + (2 if escape_open else 0) + literal_depth + ( - 1 if hex_open else 0 - ) + ( - array_depth + len(b" TJ") - if include_text_show and array_depth - else len(b" Tj") - if include_text_show and (literal_depth or hex_open or bare_string_complete) - else 0 - ) <= len(prefix): + if ( + index + + 1 + + (1 if comment_open else 0) + + (2 if escape_open else 0) + + literal_depth + + (1 if hex_open else 0) + + ( + array_depth + len(b" TJ") + if include_text_show and array_depth + else len(b" Tj") + if include_text_show and (literal_depth or hex_open or bare_string_complete) + else 0 + ) + <= sample_limit + ): best_prefix_length = index + 1 best_state = ( comment_open, @@ -1340,7 +1365,9 @@ def _pdf_truncated_text_object_prefix( prefix_length: int, ) -> bytes: prefix: Final = content[start : start + prefix_length] - closed_prefix_length, closures = _pdf_truncated_text_sample_closures(prefix, include_text_show=True) + closed_prefix_length, closures = _pdf_truncated_text_sample_closures( + prefix, include_text_show=True, maximum_length=prefix_length + ) return prefix[:closed_prefix_length] + closures + b"\nET" @@ -1367,7 +1394,7 @@ def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes: if largest_operand is not None and len(largest_operand.group(0)) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2: prefix: Final = content[record.start : largest_operand.start()] suffix: Final = content[largest_operand.end() : record.end] - object_closure: Final = b"" if suffix.endswith(b"ET") else b"\nET" + object_closure: Final = b"" if record.closed else b"\nET" operand_prefix_length: Final = ( PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(prefix) - len(suffix) - len(object_closure) ) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index de6bda1e8d0..91cf938ab4a 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2739,6 +2739,38 @@ def test_oversized_literal_cut_after_escape_stays_extractable(): assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 +def test_oversized_literal_cut_after_et_inside_string_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf (" + literal_prefix: Final = b"x" * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(text_prefix) - len(b"ET")) + b"ET" + literal_suffix: Final = b"x" * 256 + b") Tj ET" + page: Final = _fake_pdf_page(text_prefix + literal_prefix + literal_suffix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = (literal_prefix + literal_suffix[: -len(b") Tj ET")]).count(b"x") + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_unterminated_pdf_text_object_ending_with_et_inside_string_is_closed(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + page: Final = _fake_pdf_page(path_padding + b"BT /F1 12 Tf (x ET") + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert priced - blank_price == 1 + + def test_oversized_text_cut_inside_trailing_comment_stays_extractable(): text_prefix: Final = b"BT /F1 12 Tf (x) Tj " comment: Final = b"%" + b"comment" * PDF_PAGE_MAX_EXACT_CONTENT_BYTES From 5f72afb65f7a79c59517b7247d7980b349840c8e Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 02:58:07 +0000 Subject: [PATCH 19/21] fix(token_counter): treat NUL as PDF whitespace Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 53 +++++++++++++++---- .../test_token_counter_pdf_benchmarks.py | 29 +++++++--- .../litellm_core_utils/test_token_counter.py | 49 +++++++++++++++++ 3 files changed, 114 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 25b51e394b2..53e0bb1655a 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -353,25 +353,56 @@ Type for a function that counts tokens in a string. EXTRAPOLATION_SAMPLES: Final = 16 PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 -PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\s\S]|[^\\()]|\((?:\\[\s\S]|[^\\()])*\)?)*\)?" -PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*") +PDF_WHITESPACE_CLASS: Final = b"[\x00\t\n\x0c\r ]" +PDF_HEX_STRING_OPERAND: Final = rb"<[0-9A-Fa-f" + PDF_WHITESPACE_CLASS[1:-1] + rb"]*>" +PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\x00-\xff]|[^\\()]|\((?:\\[\x00-\xff]|[^\\()])*\)?)*\)?" +PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|" + PDF_HEX_STRING_OPERAND + rb"|%[^\r\n]*") PDF_SIMPLE_LITERAL_STRING: Final = re.compile(rb"\([^()]*\)") -PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\s\S])") +PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\x00-\xff])") PDF_INLINE_IMAGE: Final = re.compile( PDF_LITERAL_STRING_OPERAND - + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BI(?=[\s/\[\]()<>{}%])" - + rb"[\s\S]*?(?:(?<=\s)ID\s[\s\S]*?(?:(?P\sEI(?![^\s\[/<(%]))|\Z)|\Z))" + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|(?P(?)])BI(?=" + + PDF_WHITESPACE_CLASS + + rb"|[/\[\]()<>{}%])[\x00-\xff]*?(?:(?<=" + + PDF_WHITESPACE_CLASS + + rb")ID" + + PDF_WHITESPACE_CLASS + + rb"[\x00-\xff]*?(?:(?P" + + PDF_WHITESPACE_CLASS + + rb"EI(?![^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"\[/<(%]))|\Z)|\Z))" ) PDF_TEXT_OBJECT: Final = re.compile( PDF_LITERAL_STRING_OPERAND - + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|(?P(?)])BT(?=[\s/\[\]()<>{}%])" + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|(?P(?)])BT(?=" + + PDF_WHITESPACE_CLASS + + rb"|[/\[\]()<>{}%])" + rb"(?:(?:" + PDF_LITERAL_STRING_OPERAND - + rb"|<[0-9A-Fa-f\s]*>|%[^\r\n]*|[\s\S])*?" - + rb"(?:(?P(?<=\s)ET(?![^\s\[\]()<>{}/%]))|\Z)))" + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|[\x00-\xff])*?(?:(?P(?<=" + + PDF_WHITESPACE_CLASS + + rb")ET(?![^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"\[\]()<>{}/%]))|\Z)))" ) PDF_XOBJECT_INVOCATION: Final = re.compile( - PDF_LITERAL_STRING_OPERAND + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^\s/\[\]()<>{}%]+)(?:\s|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" + PDF_LITERAL_STRING_OPERAND + + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"/\[\]()<>{}%]+)(?:" + + PDF_WHITESPACE_CLASS + + rb"|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" ) T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -1099,7 +1130,7 @@ def _pdf_decode_literal_escape(match: re.Match[bytes]) -> bytes: def _pdf_decode_string_operand(operand: bytes) -> bytes: if operand.startswith(b"<"): - digits: Final = re.sub(rb"\s+", b"", operand[1:-1]) + digits: Final = re.sub(PDF_WHITESPACE_CLASS + rb"+", b"", operand[1:-1]) padded_digits: Final = digits if len(digits) % 2 == 0 else digits + b"0" return bytes.fromhex(padded_digits.decode("ascii")) literal: Final = operand[1:-1] if operand.endswith(b")") else operand[1:] @@ -1309,7 +1340,7 @@ def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax elif byte == ord("]"): array_depth = max(0, array_depth - 1) bare_string_complete = False - elif byte not in b" \t\r\n\f\x00": + elif byte not in PDF_WHITESPACE_CLASS[1:-1]: bare_string_complete = False if ( index diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py index 577b8a9dc3a..d7c987c0e5b 100644 --- a/tests/benchmarks/test_token_counter_pdf_benchmarks.py +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -13,12 +13,25 @@ from litellm.litellm_core_utils.token_counter import ( ) -def _pdf_page(content: bytes) -> SimpleNamespace: +class _PdfBenchmarkForm: + def get_data(self) -> bytes: + return b"q Q\n" + + def get(self, key: str) -> str | None: + return "/Form" if key == "/Subtype" else None + + def items(self) -> tuple[tuple[str, object], ...]: + return () + + +def _pdf_page(content: bytes, with_form: bool = False) -> SimpleNamespace: + xobjects: Final = {"/Fm1": _PdfBenchmarkForm()} if with_form else {} + resources: Final = {"/XObject": xobjects} return SimpleNamespace( mediabox=SimpleNamespace(width=612.0, height=792.0), get_contents=lambda: SimpleNamespace(get_data=lambda: content), extract_text=lambda: "", - get=lambda key: {"/XObject": {}} if key == "/Resources" else None, + get=lambda key: resources if key == "/Resources" else None, ) @@ -63,9 +76,11 @@ def test_pdf_string_operand_scan_after_escaped_parentheses(): # test-quality-ok @pytest.mark.benchmark def test_pdf_invocation_scan_ignores_unterminated_comments(): # test-quality-ok: CodSpeed performance benchmark - page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4)) + page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4), with_form=True) - _price_pdf_pages((page,), len) + priced: Final = _price_pdf_pages((page,), len) + + assert priced > 0 @pytest.mark.benchmark @@ -130,6 +145,8 @@ def test_pdf_inline_image_scan_no_ei(repetitions: int): # test-quality-ok: CodS ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], ) def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): # test-quality-ok: CodSpeed performance benchmark - page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET") + page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET", with_form=True) - _price_pdf_pages((page,), len) + priced: Final = _price_pdf_pages((page,), len) + + assert priced > 0 diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 91cf938ab4a..fa9bd8cad8f 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2907,6 +2907,20 @@ def test_compact_bt_operator_keeps_oversized_page_text(): assert priced - blank_price == len("Hello world") +def test_nul_whitespace_after_bt_keeps_oversized_page_text(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + content: Final = path_padding + b"BT\x00/F1 12 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + def test_compact_bi_operator_strips_inline_image_before_pricing_text(): image_data: Final = random.Random(42).randbytes(270_000) content: Final = b"BI/W 270000/H 1/BPC 8/CS /RGB ID " + image_data + b" EI\nBT /F1 9 Tf (Hello world) Tj ET\n" @@ -2962,6 +2976,41 @@ def test_repeated_identity_h_form_draws_count_each_text_sample(): assert priced - blank_price == 2 * len("Hello world") +def test_nul_whitespace_invokes_large_form_with_bounded_sample_pricing(): + text_object: Final = b"BT /F1 12 Tf (Hello world) Tj ET\n" + repetitions: Final = 8 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_object) + 1 + form_object: Final = _pdf_stream( + text_object * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + nul_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1\x00Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + space_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + nul_count_function: Final = MagicMock(side_effect=len) + space_count_function: Final = MagicMock(side_effect=len) + + nul_price: Final = _count_inline_pdf_tokens(nul_data_url, nul_count_function) + space_price: Final = _count_inline_pdf_tokens(space_data_url, space_count_function) + + assert nul_price is not None + assert space_price is not None + assert nul_price == space_price + assert nul_price > 0 + assert max((len(call.args[0]) for call in nul_count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert max((len(call.args[0]) for call in space_count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): count_function: Final = MagicMock(side_effect=len) sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " From 8a527fc3bb2ca793043dab542008d59a52ea5035 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 03:29:38 +0000 Subject: [PATCH 20/21] test(token_counter): cover octal PDF sample scaling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../unit/litellm_core_utils/test_token_counter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index fa9bd8cad8f..70f2485691e 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2687,6 +2687,20 @@ def test_oversized_hex_pdf_strings_price_like_literal_text(): assert literal_text_tokens * 5 <= hex_text_tokens * 6 +def test_oversized_pdf_octal_escapes_keep_sample_scaling_by_decoded_bytes(): + plain_count: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2 + 2 + octal_count: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2 + 2 + text_object: Final = b"BT /F1 12 Tf (" + b"A" * plain_count + b"\\101" * octal_count + b") Tj ET\n" + page: Final = _fake_pdf_page(content_bytes=text_object) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + priced: Final = _price_pdf_pages((page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + expected: Final = plain_count + octal_count + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + def test_oversized_pdf_text_uses_a_sampled_token_estimate(): string_operand: Final = b"(Revenue grew eleven percent)" text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" From 658ca5c9a90d3f3dcf4556853a72e1815911660e Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 21 Oct 2026 04:35:44 +0000 Subject: [PATCH 21/21] fix(token_counter): repair dangling escapes and keep sampling past unclosed text Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 4 +- .../litellm_core_utils/test_token_counter.py | 51 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 53e0bb1655a..e7b1ca2b756 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1193,7 +1193,7 @@ def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream P yield ( match.start("text_object"), match.end("text_object"), - match.group("text_object_end") is None, + False, match.group("text_object_end") is not None, ) @@ -1376,7 +1376,7 @@ def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax best_bare_string_complete, ) = best_state operand_closures: Final = ( - (b"\\\\" if best_escape_open else b"") + (b"\\" if best_escape_open else b"") + (b")" * best_literal_depth if best_literal_depth else b"") + (b">" if best_hex_open else b"") ) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 70f2485691e..b4caf18a385 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -2753,6 +2753,57 @@ def test_oversized_literal_cut_after_escape_stays_extractable(): assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 +@pytest.mark.parametrize("backslash_count", [1, 2, 3]) +def test_oversized_unclosed_literal_with_backslash_run_stays_extractable(backslash_count: int): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) - 1) + literal_prefix: Final = b"BT /F1 12 Tf (escaped \\(paren\\) " + b"x" * 512 + b"\\" * backslash_count + closed_suffix: Final = b")" * (1 + backslash_count % 2) + b" Tj ET\n" + closed_page: Final = _fake_pdf_page(path_padding + literal_prefix + closed_suffix) + unclosed_page: Final = _fake_pdf_page(path_padding + literal_prefix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + closed_price: Final = _price_pdf_pages((closed_page,), count_x_tokens) - blank_price + unclosed_price: Final = _price_pdf_pages((unclosed_page,), count_x_tokens) - blank_price + expected: Final = literal_prefix.count(b"x") + + assert expected * 3 // 4 <= closed_price <= expected * 5 // 4 + assert expected * 3 // 4 <= unclosed_price <= expected * 5 // 4 + + +def test_unclosed_end_of_stream_text_keeps_sampling_form_text(): + form_text: Final = b"BT /F1 12 Tf (form text) Tj ET\n" + form_repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(form_text) + 1 + form_object: Final = _pdf_stream( + form_text * form_repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + unclosed_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm Do\nBT /F1 12 Tf (hi) Tj", + b"<< /Fm 6 0 R >>", + (form_object,), + ) + closed_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm Do\nBT /F1 12 Tf (hi) Tj ET\n", + b"<< /Fm 6 0 R >>", + (form_object,), + ) + + def count_form_tokens(value: str) -> int: + return value.count("form") + + unclosed_price: Final = _count_inline_pdf_tokens(unclosed_data_url, count_form_tokens) + closed_price: Final = _count_inline_pdf_tokens(closed_data_url, count_form_tokens) + + assert unclosed_price is not None + assert closed_price is not None + assert closed_price * 3 // 4 <= unclosed_price <= closed_price * 5 // 4 + + def test_oversized_literal_cut_after_et_inside_string_stays_extractable(): text_prefix: Final = b"BT /F1 12 Tf (" literal_prefix: Final = b"x" * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(text_prefix) - len(b"ET")) + b"ET"